Index: stable/12/contrib/llvm-project/clang/lib/AST/ExprConstant.cpp =================================================================== --- stable/12/contrib/llvm-project/clang/lib/AST/ExprConstant.cpp (revision 356466) +++ stable/12/contrib/llvm-project/clang/lib/AST/ExprConstant.cpp (revision 356467) @@ -1,13225 +1,13232 @@ //===--- ExprConstant.cpp - Expression Constant Evaluator -----------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file implements the Expr constant evaluator. // // Constant expression evaluation produces four main results: // // * A success/failure flag indicating whether constant folding was successful. // This is the 'bool' return value used by most of the code in this file. A // 'false' return value indicates that constant folding has failed, and any // appropriate diagnostic has already been produced. // // * An evaluated result, valid only if constant folding has not failed. // // * A flag indicating if evaluation encountered (unevaluated) side-effects. // These arise in cases such as (sideEffect(), 0) and (sideEffect() || 1), // where it is possible to determine the evaluated result regardless. // // * A set of notes indicating why the evaluation was not a constant expression // (under the C++11 / C++1y rules only, at the moment), or, if folding failed // too, why the expression could not be folded. // // If we are checking for a potential constant expression, failure to constant // fold a potential constant sub-expression will be indicated by a 'false' // return value (the expression could not be folded) and no diagnostic (the // expression is not necessarily non-constant). // //===----------------------------------------------------------------------===// #include "clang/AST/APValue.h" #include "clang/AST/ASTContext.h" #include "clang/AST/ASTDiagnostic.h" #include "clang/AST/ASTLambda.h" #include "clang/AST/CharUnits.h" #include "clang/AST/CurrentSourceLocExprScope.h" #include "clang/AST/CXXInheritance.h" #include "clang/AST/Expr.h" #include "clang/AST/OSLog.h" #include "clang/AST/RecordLayout.h" #include "clang/AST/StmtVisitor.h" #include "clang/AST/TypeLoc.h" #include "clang/Basic/Builtins.h" #include "clang/Basic/FixedPoint.h" #include "clang/Basic/TargetInfo.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallBitVector.h" #include "llvm/Support/SaveAndRestore.h" #include "llvm/Support/raw_ostream.h" #include #include #define DEBUG_TYPE "exprconstant" using namespace clang; using llvm::APInt; using llvm::APSInt; using llvm::APFloat; using llvm::Optional; static bool IsGlobalLValue(APValue::LValueBase B); namespace { struct LValue; struct CallStackFrame; struct EvalInfo; using SourceLocExprScopeGuard = CurrentSourceLocExprScope::SourceLocExprScopeGuard; static QualType getType(APValue::LValueBase B) { if (!B) return QualType(); if (const ValueDecl *D = B.dyn_cast()) { // FIXME: It's unclear where we're supposed to take the type from, and // this actually matters for arrays of unknown bound. Eg: // // extern int arr[]; void f() { extern int arr[3]; }; // constexpr int *p = &arr[1]; // valid? // // For now, we take the array bound from the most recent declaration. for (auto *Redecl = cast(D->getMostRecentDecl()); Redecl; Redecl = cast_or_null(Redecl->getPreviousDecl())) { QualType T = Redecl->getType(); if (!T->isIncompleteArrayType()) return T; } return D->getType(); } if (B.is()) return B.getTypeInfoType(); const Expr *Base = B.get(); // For a materialized temporary, the type of the temporary we materialized // may not be the type of the expression. if (const MaterializeTemporaryExpr *MTE = dyn_cast(Base)) { SmallVector CommaLHSs; SmallVector Adjustments; const Expr *Temp = MTE->GetTemporaryExpr(); const Expr *Inner = Temp->skipRValueSubobjectAdjustments(CommaLHSs, Adjustments); // Keep any cv-qualifiers from the reference if we generated a temporary // for it directly. Otherwise use the type after adjustment. if (!Adjustments.empty()) return Inner->getType(); } return Base->getType(); } /// Get an LValue path entry, which is known to not be an array index, as a /// field declaration. static const FieldDecl *getAsField(APValue::LValuePathEntry E) { return dyn_cast_or_null(E.getAsBaseOrMember().getPointer()); } /// Get an LValue path entry, which is known to not be an array index, as a /// base class declaration. static const CXXRecordDecl *getAsBaseClass(APValue::LValuePathEntry E) { return dyn_cast_or_null(E.getAsBaseOrMember().getPointer()); } /// Determine whether this LValue path entry for a base class names a virtual /// base class. static bool isVirtualBaseClass(APValue::LValuePathEntry E) { return E.getAsBaseOrMember().getInt(); } /// Given a CallExpr, try to get the alloc_size attribute. May return null. static const AllocSizeAttr *getAllocSizeAttr(const CallExpr *CE) { const FunctionDecl *Callee = CE->getDirectCallee(); return Callee ? Callee->getAttr() : nullptr; } /// Attempts to unwrap a CallExpr (with an alloc_size attribute) from an Expr. /// This will look through a single cast. /// /// Returns null if we couldn't unwrap a function with alloc_size. static const CallExpr *tryUnwrapAllocSizeCall(const Expr *E) { if (!E->getType()->isPointerType()) return nullptr; E = E->IgnoreParens(); // If we're doing a variable assignment from e.g. malloc(N), there will // probably be a cast of some kind. In exotic cases, we might also see a // top-level ExprWithCleanups. Ignore them either way. if (const auto *FE = dyn_cast(E)) E = FE->getSubExpr()->IgnoreParens(); if (const auto *Cast = dyn_cast(E)) E = Cast->getSubExpr()->IgnoreParens(); if (const auto *CE = dyn_cast(E)) return getAllocSizeAttr(CE) ? CE : nullptr; return nullptr; } /// Determines whether or not the given Base contains a call to a function /// with the alloc_size attribute. static bool isBaseAnAllocSizeCall(APValue::LValueBase Base) { const auto *E = Base.dyn_cast(); return E && E->getType()->isPointerType() && tryUnwrapAllocSizeCall(E); } /// The bound to claim that an array of unknown bound has. /// The value in MostDerivedArraySize is undefined in this case. So, set it /// to an arbitrary value that's likely to loudly break things if it's used. static const uint64_t AssumedSizeForUnsizedArray = std::numeric_limits::max() / 2; /// Determines if an LValue with the given LValueBase will have an unsized /// array in its designator. /// Find the path length and type of the most-derived subobject in the given /// path, and find the size of the containing array, if any. static unsigned findMostDerivedSubobject(ASTContext &Ctx, APValue::LValueBase Base, ArrayRef Path, uint64_t &ArraySize, QualType &Type, bool &IsArray, bool &FirstEntryIsUnsizedArray) { // This only accepts LValueBases from APValues, and APValues don't support // arrays that lack size info. assert(!isBaseAnAllocSizeCall(Base) && "Unsized arrays shouldn't appear here"); unsigned MostDerivedLength = 0; Type = getType(Base); for (unsigned I = 0, N = Path.size(); I != N; ++I) { if (Type->isArrayType()) { const ArrayType *AT = Ctx.getAsArrayType(Type); Type = AT->getElementType(); MostDerivedLength = I + 1; IsArray = true; if (auto *CAT = dyn_cast(AT)) { ArraySize = CAT->getSize().getZExtValue(); } else { assert(I == 0 && "unexpected unsized array designator"); FirstEntryIsUnsizedArray = true; ArraySize = AssumedSizeForUnsizedArray; } } else if (Type->isAnyComplexType()) { const ComplexType *CT = Type->castAs(); Type = CT->getElementType(); ArraySize = 2; MostDerivedLength = I + 1; IsArray = true; } else if (const FieldDecl *FD = getAsField(Path[I])) { Type = FD->getType(); ArraySize = 0; MostDerivedLength = I + 1; IsArray = false; } else { // Path[I] describes a base class. ArraySize = 0; IsArray = false; } } return MostDerivedLength; } // The order of this enum is important for diagnostics. enum CheckSubobjectKind { CSK_Base, CSK_Derived, CSK_Field, CSK_ArrayToPointer, CSK_ArrayIndex, CSK_Real, CSK_Imag }; /// A path from a glvalue to a subobject of that glvalue. struct SubobjectDesignator { /// True if the subobject was named in a manner not supported by C++11. Such /// lvalues can still be folded, but they are not core constant expressions /// and we cannot perform lvalue-to-rvalue conversions on them. unsigned Invalid : 1; /// Is this a pointer one past the end of an object? unsigned IsOnePastTheEnd : 1; /// Indicator of whether the first entry is an unsized array. unsigned FirstEntryIsAnUnsizedArray : 1; /// Indicator of whether the most-derived object is an array element. unsigned MostDerivedIsArrayElement : 1; /// The length of the path to the most-derived object of which this is a /// subobject. unsigned MostDerivedPathLength : 28; /// The size of the array of which the most-derived object is an element. /// This will always be 0 if the most-derived object is not an array /// element. 0 is not an indicator of whether or not the most-derived object /// is an array, however, because 0-length arrays are allowed. /// /// If the current array is an unsized array, the value of this is /// undefined. uint64_t MostDerivedArraySize; /// The type of the most derived object referred to by this address. QualType MostDerivedType; typedef APValue::LValuePathEntry PathEntry; /// The entries on the path from the glvalue to the designated subobject. SmallVector Entries; SubobjectDesignator() : Invalid(true) {} explicit SubobjectDesignator(QualType T) : Invalid(false), IsOnePastTheEnd(false), FirstEntryIsAnUnsizedArray(false), MostDerivedIsArrayElement(false), MostDerivedPathLength(0), MostDerivedArraySize(0), MostDerivedType(T) {} SubobjectDesignator(ASTContext &Ctx, const APValue &V) : Invalid(!V.isLValue() || !V.hasLValuePath()), IsOnePastTheEnd(false), FirstEntryIsAnUnsizedArray(false), MostDerivedIsArrayElement(false), MostDerivedPathLength(0), MostDerivedArraySize(0) { assert(V.isLValue() && "Non-LValue used to make an LValue designator?"); if (!Invalid) { IsOnePastTheEnd = V.isLValueOnePastTheEnd(); ArrayRef VEntries = V.getLValuePath(); Entries.insert(Entries.end(), VEntries.begin(), VEntries.end()); if (V.getLValueBase()) { bool IsArray = false; bool FirstIsUnsizedArray = false; MostDerivedPathLength = findMostDerivedSubobject( Ctx, V.getLValueBase(), V.getLValuePath(), MostDerivedArraySize, MostDerivedType, IsArray, FirstIsUnsizedArray); MostDerivedIsArrayElement = IsArray; FirstEntryIsAnUnsizedArray = FirstIsUnsizedArray; } } } void truncate(ASTContext &Ctx, APValue::LValueBase Base, unsigned NewLength) { if (Invalid) return; assert(Base && "cannot truncate path for null pointer"); assert(NewLength <= Entries.size() && "not a truncation"); if (NewLength == Entries.size()) return; Entries.resize(NewLength); bool IsArray = false; bool FirstIsUnsizedArray = false; MostDerivedPathLength = findMostDerivedSubobject( Ctx, Base, Entries, MostDerivedArraySize, MostDerivedType, IsArray, FirstIsUnsizedArray); MostDerivedIsArrayElement = IsArray; FirstEntryIsAnUnsizedArray = FirstIsUnsizedArray; } void setInvalid() { Invalid = true; Entries.clear(); } /// Determine whether the most derived subobject is an array without a /// known bound. bool isMostDerivedAnUnsizedArray() const { assert(!Invalid && "Calling this makes no sense on invalid designators"); return Entries.size() == 1 && FirstEntryIsAnUnsizedArray; } /// Determine what the most derived array's size is. Results in an assertion /// failure if the most derived array lacks a size. uint64_t getMostDerivedArraySize() const { assert(!isMostDerivedAnUnsizedArray() && "Unsized array has no size"); return MostDerivedArraySize; } /// Determine whether this is a one-past-the-end pointer. bool isOnePastTheEnd() const { assert(!Invalid); if (IsOnePastTheEnd) return true; if (!isMostDerivedAnUnsizedArray() && MostDerivedIsArrayElement && Entries[MostDerivedPathLength - 1].getAsArrayIndex() == MostDerivedArraySize) return true; return false; } /// Get the range of valid index adjustments in the form /// {maximum value that can be subtracted from this pointer, /// maximum value that can be added to this pointer} std::pair validIndexAdjustments() { if (Invalid || isMostDerivedAnUnsizedArray()) return {0, 0}; // [expr.add]p4: For the purposes of these operators, a pointer to a // nonarray object behaves the same as a pointer to the first element of // an array of length one with the type of the object as its element type. bool IsArray = MostDerivedPathLength == Entries.size() && MostDerivedIsArrayElement; uint64_t ArrayIndex = IsArray ? Entries.back().getAsArrayIndex() : (uint64_t)IsOnePastTheEnd; uint64_t ArraySize = IsArray ? getMostDerivedArraySize() : (uint64_t)1; return {ArrayIndex, ArraySize - ArrayIndex}; } /// Check that this refers to a valid subobject. bool isValidSubobject() const { if (Invalid) return false; return !isOnePastTheEnd(); } /// Check that this refers to a valid subobject, and if not, produce a /// relevant diagnostic and set the designator as invalid. bool checkSubobject(EvalInfo &Info, const Expr *E, CheckSubobjectKind CSK); /// Get the type of the designated object. QualType getType(ASTContext &Ctx) const { assert(!Invalid && "invalid designator has no subobject type"); return MostDerivedPathLength == Entries.size() ? MostDerivedType : Ctx.getRecordType(getAsBaseClass(Entries.back())); } /// Update this designator to refer to the first element within this array. void addArrayUnchecked(const ConstantArrayType *CAT) { Entries.push_back(PathEntry::ArrayIndex(0)); // This is a most-derived object. MostDerivedType = CAT->getElementType(); MostDerivedIsArrayElement = true; MostDerivedArraySize = CAT->getSize().getZExtValue(); MostDerivedPathLength = Entries.size(); } /// Update this designator to refer to the first element within the array of /// elements of type T. This is an array of unknown size. void addUnsizedArrayUnchecked(QualType ElemTy) { Entries.push_back(PathEntry::ArrayIndex(0)); MostDerivedType = ElemTy; MostDerivedIsArrayElement = true; // The value in MostDerivedArraySize is undefined in this case. So, set it // to an arbitrary value that's likely to loudly break things if it's // used. MostDerivedArraySize = AssumedSizeForUnsizedArray; MostDerivedPathLength = Entries.size(); } /// Update this designator to refer to the given base or member of this /// object. void addDeclUnchecked(const Decl *D, bool Virtual = false) { Entries.push_back(APValue::BaseOrMemberType(D, Virtual)); // If this isn't a base class, it's a new most-derived object. if (const FieldDecl *FD = dyn_cast(D)) { MostDerivedType = FD->getType(); MostDerivedIsArrayElement = false; MostDerivedArraySize = 0; MostDerivedPathLength = Entries.size(); } } /// Update this designator to refer to the given complex component. void addComplexUnchecked(QualType EltTy, bool Imag) { Entries.push_back(PathEntry::ArrayIndex(Imag)); // This is technically a most-derived object, though in practice this // is unlikely to matter. MostDerivedType = EltTy; MostDerivedIsArrayElement = true; MostDerivedArraySize = 2; MostDerivedPathLength = Entries.size(); } void diagnoseUnsizedArrayPointerArithmetic(EvalInfo &Info, const Expr *E); void diagnosePointerArithmetic(EvalInfo &Info, const Expr *E, const APSInt &N); /// Add N to the address of this subobject. void adjustIndex(EvalInfo &Info, const Expr *E, APSInt N) { if (Invalid || !N) return; uint64_t TruncatedN = N.extOrTrunc(64).getZExtValue(); if (isMostDerivedAnUnsizedArray()) { diagnoseUnsizedArrayPointerArithmetic(Info, E); // Can't verify -- trust that the user is doing the right thing (or if // not, trust that the caller will catch the bad behavior). // FIXME: Should we reject if this overflows, at least? Entries.back() = PathEntry::ArrayIndex( Entries.back().getAsArrayIndex() + TruncatedN); return; } // [expr.add]p4: For the purposes of these operators, a pointer to a // nonarray object behaves the same as a pointer to the first element of // an array of length one with the type of the object as its element type. bool IsArray = MostDerivedPathLength == Entries.size() && MostDerivedIsArrayElement; uint64_t ArrayIndex = IsArray ? Entries.back().getAsArrayIndex() : (uint64_t)IsOnePastTheEnd; uint64_t ArraySize = IsArray ? getMostDerivedArraySize() : (uint64_t)1; if (N < -(int64_t)ArrayIndex || N > ArraySize - ArrayIndex) { // Calculate the actual index in a wide enough type, so we can include // it in the note. N = N.extend(std::max(N.getBitWidth() + 1, 65)); (llvm::APInt&)N += ArrayIndex; assert(N.ugt(ArraySize) && "bounds check failed for in-bounds index"); diagnosePointerArithmetic(Info, E, N); setInvalid(); return; } ArrayIndex += TruncatedN; assert(ArrayIndex <= ArraySize && "bounds check succeeded for out-of-bounds index"); if (IsArray) Entries.back() = PathEntry::ArrayIndex(ArrayIndex); else IsOnePastTheEnd = (ArrayIndex != 0); } }; /// A stack frame in the constexpr call stack. struct CallStackFrame { EvalInfo &Info; /// Parent - The caller of this stack frame. CallStackFrame *Caller; /// Callee - The function which was called. const FunctionDecl *Callee; /// This - The binding for the this pointer in this call, if any. const LValue *This; /// Arguments - Parameter bindings for this function call, indexed by /// parameters' function scope indices. APValue *Arguments; /// Source location information about the default argument or default /// initializer expression we're evaluating, if any. CurrentSourceLocExprScope CurSourceLocExprScope; // Note that we intentionally use std::map here so that references to // values are stable. typedef std::pair MapKeyTy; typedef std::map MapTy; /// Temporaries - Temporary lvalues materialized within this stack frame. MapTy Temporaries; /// CallLoc - The location of the call expression for this call. SourceLocation CallLoc; /// Index - The call index of this call. unsigned Index; /// The stack of integers for tracking version numbers for temporaries. SmallVector TempVersionStack = {1}; unsigned CurTempVersion = TempVersionStack.back(); unsigned getTempVersion() const { return TempVersionStack.back(); } void pushTempVersion() { TempVersionStack.push_back(++CurTempVersion); } void popTempVersion() { TempVersionStack.pop_back(); } // FIXME: Adding this to every 'CallStackFrame' may have a nontrivial impact // on the overall stack usage of deeply-recursing constexpr evaluations. // (We should cache this map rather than recomputing it repeatedly.) // But let's try this and see how it goes; we can look into caching the map // as a later change. /// LambdaCaptureFields - Mapping from captured variables/this to /// corresponding data members in the closure class. llvm::DenseMap LambdaCaptureFields; FieldDecl *LambdaThisCaptureField; CallStackFrame(EvalInfo &Info, SourceLocation CallLoc, const FunctionDecl *Callee, const LValue *This, APValue *Arguments); ~CallStackFrame(); // Return the temporary for Key whose version number is Version. APValue *getTemporary(const void *Key, unsigned Version) { MapKeyTy KV(Key, Version); auto LB = Temporaries.lower_bound(KV); if (LB != Temporaries.end() && LB->first == KV) return &LB->second; // Pair (Key,Version) wasn't found in the map. Check that no elements // in the map have 'Key' as their key. assert((LB == Temporaries.end() || LB->first.first != Key) && (LB == Temporaries.begin() || std::prev(LB)->first.first != Key) && "Element with key 'Key' found in map"); return nullptr; } // Return the current temporary for Key in the map. APValue *getCurrentTemporary(const void *Key) { auto UB = Temporaries.upper_bound(MapKeyTy(Key, UINT_MAX)); if (UB != Temporaries.begin() && std::prev(UB)->first.first == Key) return &std::prev(UB)->second; return nullptr; } // Return the version number of the current temporary for Key. unsigned getCurrentTemporaryVersion(const void *Key) const { auto UB = Temporaries.upper_bound(MapKeyTy(Key, UINT_MAX)); if (UB != Temporaries.begin() && std::prev(UB)->first.first == Key) return std::prev(UB)->first.second; return 0; } APValue &createTemporary(const void *Key, bool IsLifetimeExtended); }; /// Temporarily override 'this'. class ThisOverrideRAII { public: ThisOverrideRAII(CallStackFrame &Frame, const LValue *NewThis, bool Enable) : Frame(Frame), OldThis(Frame.This) { if (Enable) Frame.This = NewThis; } ~ThisOverrideRAII() { Frame.This = OldThis; } private: CallStackFrame &Frame; const LValue *OldThis; }; /// A partial diagnostic which we might know in advance that we are not going /// to emit. class OptionalDiagnostic { PartialDiagnostic *Diag; public: explicit OptionalDiagnostic(PartialDiagnostic *Diag = nullptr) : Diag(Diag) {} template OptionalDiagnostic &operator<<(const T &v) { if (Diag) *Diag << v; return *this; } OptionalDiagnostic &operator<<(const APSInt &I) { if (Diag) { SmallVector Buffer; I.toString(Buffer); *Diag << StringRef(Buffer.data(), Buffer.size()); } return *this; } OptionalDiagnostic &operator<<(const APFloat &F) { if (Diag) { // FIXME: Force the precision of the source value down so we don't // print digits which are usually useless (we don't really care here if // we truncate a digit by accident in edge cases). Ideally, // APFloat::toString would automatically print the shortest // representation which rounds to the correct value, but it's a bit // tricky to implement. unsigned precision = llvm::APFloat::semanticsPrecision(F.getSemantics()); precision = (precision * 59 + 195) / 196; SmallVector Buffer; F.toString(Buffer, precision); *Diag << StringRef(Buffer.data(), Buffer.size()); } return *this; } OptionalDiagnostic &operator<<(const APFixedPoint &FX) { if (Diag) { SmallVector Buffer; FX.toString(Buffer); *Diag << StringRef(Buffer.data(), Buffer.size()); } return *this; } }; /// A cleanup, and a flag indicating whether it is lifetime-extended. class Cleanup { llvm::PointerIntPair Value; public: Cleanup(APValue *Val, bool IsLifetimeExtended) : Value(Val, IsLifetimeExtended) {} bool isLifetimeExtended() const { return Value.getInt(); } void endLifetime() { *Value.getPointer() = APValue(); } }; /// A reference to an object whose construction we are currently evaluating. struct ObjectUnderConstruction { APValue::LValueBase Base; ArrayRef Path; friend bool operator==(const ObjectUnderConstruction &LHS, const ObjectUnderConstruction &RHS) { return LHS.Base == RHS.Base && LHS.Path == RHS.Path; } friend llvm::hash_code hash_value(const ObjectUnderConstruction &Obj) { return llvm::hash_combine(Obj.Base, Obj.Path); } }; enum class ConstructionPhase { None, Bases, AfterBases }; } namespace llvm { template<> struct DenseMapInfo { using Base = DenseMapInfo; static ObjectUnderConstruction getEmptyKey() { return {Base::getEmptyKey(), {}}; } static ObjectUnderConstruction getTombstoneKey() { return {Base::getTombstoneKey(), {}}; } static unsigned getHashValue(const ObjectUnderConstruction &Object) { return hash_value(Object); } static bool isEqual(const ObjectUnderConstruction &LHS, const ObjectUnderConstruction &RHS) { return LHS == RHS; } }; } namespace { /// EvalInfo - This is a private struct used by the evaluator to capture /// information about a subexpression as it is folded. It retains information /// about the AST context, but also maintains information about the folded /// expression. /// /// If an expression could be evaluated, it is still possible it is not a C /// "integer constant expression" or constant expression. If not, this struct /// captures information about how and why not. /// /// One bit of information passed *into* the request for constant folding /// indicates whether the subexpression is "evaluated" or not according to C /// rules. For example, the RHS of (0 && foo()) is not evaluated. We can /// evaluate the expression regardless of what the RHS is, but C only allows /// certain things in certain situations. struct EvalInfo { ASTContext &Ctx; /// EvalStatus - Contains information about the evaluation. Expr::EvalStatus &EvalStatus; /// CurrentCall - The top of the constexpr call stack. CallStackFrame *CurrentCall; /// CallStackDepth - The number of calls in the call stack right now. unsigned CallStackDepth; /// NextCallIndex - The next call index to assign. unsigned NextCallIndex; /// StepsLeft - The remaining number of evaluation steps we're permitted /// to perform. This is essentially a limit for the number of statements /// we will evaluate. unsigned StepsLeft; /// BottomFrame - The frame in which evaluation started. This must be /// initialized after CurrentCall and CallStackDepth. CallStackFrame BottomFrame; /// A stack of values whose lifetimes end at the end of some surrounding /// evaluation frame. llvm::SmallVector CleanupStack; /// EvaluatingDecl - This is the declaration whose initializer is being /// evaluated, if any. APValue::LValueBase EvaluatingDecl; /// EvaluatingDeclValue - This is the value being constructed for the /// declaration whose initializer is being evaluated, if any. APValue *EvaluatingDeclValue; /// Set of objects that are currently being constructed. llvm::DenseMap ObjectsUnderConstruction; struct EvaluatingConstructorRAII { EvalInfo &EI; ObjectUnderConstruction Object; bool DidInsert; EvaluatingConstructorRAII(EvalInfo &EI, ObjectUnderConstruction Object, bool HasBases) : EI(EI), Object(Object) { DidInsert = EI.ObjectsUnderConstruction .insert({Object, HasBases ? ConstructionPhase::Bases : ConstructionPhase::AfterBases}) .second; } void finishedConstructingBases() { EI.ObjectsUnderConstruction[Object] = ConstructionPhase::AfterBases; } ~EvaluatingConstructorRAII() { if (DidInsert) EI.ObjectsUnderConstruction.erase(Object); } }; ConstructionPhase isEvaluatingConstructor(APValue::LValueBase Base, ArrayRef Path) { return ObjectsUnderConstruction.lookup({Base, Path}); } /// If we're currently speculatively evaluating, the outermost call stack /// depth at which we can mutate state, otherwise 0. unsigned SpeculativeEvaluationDepth = 0; /// The current array initialization index, if we're performing array /// initialization. uint64_t ArrayInitIndex = -1; /// HasActiveDiagnostic - Was the previous diagnostic stored? If so, further /// notes attached to it will also be stored, otherwise they will not be. bool HasActiveDiagnostic; /// Have we emitted a diagnostic explaining why we couldn't constant /// fold (not just why it's not strictly a constant expression)? bool HasFoldFailureDiagnostic; /// Whether or not we're in a context where the front end requires a /// constant value. bool InConstantContext; /// Whether we're checking that an expression is a potential constant /// expression. If so, do not fail on constructs that could become constant /// later on (such as a use of an undefined global). bool CheckingPotentialConstantExpression = false; /// Whether we're checking for an expression that has undefined behavior. /// If so, we will produce warnings if we encounter an operation that is /// always undefined. bool CheckingForUndefinedBehavior = false; enum EvaluationMode { /// Evaluate as a constant expression. Stop if we find that the expression /// is not a constant expression. EM_ConstantExpression, /// Evaluate as a constant expression. Stop if we find that the expression /// is not a constant expression. Some expressions can be retried in the /// optimizer if we don't constant fold them here, but in an unevaluated /// context we try to fold them immediately since the optimizer never /// gets a chance to look at it. EM_ConstantExpressionUnevaluated, /// Fold the expression to a constant. Stop if we hit a side-effect that /// we can't model. EM_ConstantFold, /// Evaluate in any way we know how. Don't worry about side-effects that /// can't be modeled. EM_IgnoreSideEffects, } EvalMode; /// Are we checking whether the expression is a potential constant /// expression? bool checkingPotentialConstantExpression() const { return CheckingPotentialConstantExpression; } /// Are we checking an expression for overflow? // FIXME: We should check for any kind of undefined or suspicious behavior // in such constructs, not just overflow. bool checkingForUndefinedBehavior() { return CheckingForUndefinedBehavior; } EvalInfo(const ASTContext &C, Expr::EvalStatus &S, EvaluationMode Mode) : Ctx(const_cast(C)), EvalStatus(S), CurrentCall(nullptr), CallStackDepth(0), NextCallIndex(1), StepsLeft(getLangOpts().ConstexprStepLimit), BottomFrame(*this, SourceLocation(), nullptr, nullptr, nullptr), EvaluatingDecl((const ValueDecl *)nullptr), EvaluatingDeclValue(nullptr), HasActiveDiagnostic(false), HasFoldFailureDiagnostic(false), InConstantContext(false), EvalMode(Mode) {} void setEvaluatingDecl(APValue::LValueBase Base, APValue &Value) { EvaluatingDecl = Base; EvaluatingDeclValue = &Value; } const LangOptions &getLangOpts() const { return Ctx.getLangOpts(); } bool CheckCallLimit(SourceLocation Loc) { // Don't perform any constexpr calls (other than the call we're checking) // when checking a potential constant expression. if (checkingPotentialConstantExpression() && CallStackDepth > 1) return false; if (NextCallIndex == 0) { // NextCallIndex has wrapped around. FFDiag(Loc, diag::note_constexpr_call_limit_exceeded); return false; } if (CallStackDepth <= getLangOpts().ConstexprCallDepth) return true; FFDiag(Loc, diag::note_constexpr_depth_limit_exceeded) << getLangOpts().ConstexprCallDepth; return false; } std::pair getCallFrameAndDepth(unsigned CallIndex) { assert(CallIndex && "no call index in getCallFrameAndDepth"); // We will eventually hit BottomFrame, which has Index 1, so Frame can't // be null in this loop. unsigned Depth = CallStackDepth; CallStackFrame *Frame = CurrentCall; while (Frame->Index > CallIndex) { Frame = Frame->Caller; --Depth; } if (Frame->Index == CallIndex) return {Frame, Depth}; return {nullptr, 0}; } bool nextStep(const Stmt *S) { if (!StepsLeft) { FFDiag(S->getBeginLoc(), diag::note_constexpr_step_limit_exceeded); return false; } --StepsLeft; return true; } private: /// Add a diagnostic to the diagnostics list. PartialDiagnostic &addDiag(SourceLocation Loc, diag::kind DiagId) { PartialDiagnostic PD(DiagId, Ctx.getDiagAllocator()); EvalStatus.Diag->push_back(std::make_pair(Loc, PD)); return EvalStatus.Diag->back().second; } /// Add notes containing a call stack to the current point of evaluation. void addCallStack(unsigned Limit); private: OptionalDiagnostic Diag(SourceLocation Loc, diag::kind DiagId, unsigned ExtraNotes, bool IsCCEDiag) { if (EvalStatus.Diag) { // If we have a prior diagnostic, it will be noting that the expression // isn't a constant expression. This diagnostic is more important, // unless we require this evaluation to produce a constant expression. // // FIXME: We might want to show both diagnostics to the user in // EM_ConstantFold mode. if (!EvalStatus.Diag->empty()) { switch (EvalMode) { case EM_ConstantFold: case EM_IgnoreSideEffects: if (!HasFoldFailureDiagnostic) break; // We've already failed to fold something. Keep that diagnostic. LLVM_FALLTHROUGH; case EM_ConstantExpression: case EM_ConstantExpressionUnevaluated: HasActiveDiagnostic = false; return OptionalDiagnostic(); } } unsigned CallStackNotes = CallStackDepth - 1; unsigned Limit = Ctx.getDiagnostics().getConstexprBacktraceLimit(); if (Limit) CallStackNotes = std::min(CallStackNotes, Limit + 1); if (checkingPotentialConstantExpression()) CallStackNotes = 0; HasActiveDiagnostic = true; HasFoldFailureDiagnostic = !IsCCEDiag; EvalStatus.Diag->clear(); EvalStatus.Diag->reserve(1 + ExtraNotes + CallStackNotes); addDiag(Loc, DiagId); if (!checkingPotentialConstantExpression()) addCallStack(Limit); return OptionalDiagnostic(&(*EvalStatus.Diag)[0].second); } HasActiveDiagnostic = false; return OptionalDiagnostic(); } public: // Diagnose that the evaluation could not be folded (FF => FoldFailure) OptionalDiagnostic FFDiag(SourceLocation Loc, diag::kind DiagId = diag::note_invalid_subexpr_in_const_expr, unsigned ExtraNotes = 0) { return Diag(Loc, DiagId, ExtraNotes, false); } OptionalDiagnostic FFDiag(const Expr *E, diag::kind DiagId = diag::note_invalid_subexpr_in_const_expr, unsigned ExtraNotes = 0) { if (EvalStatus.Diag) return Diag(E->getExprLoc(), DiagId, ExtraNotes, /*IsCCEDiag*/false); HasActiveDiagnostic = false; return OptionalDiagnostic(); } /// Diagnose that the evaluation does not produce a C++11 core constant /// expression. /// /// FIXME: Stop evaluating if we're in EM_ConstantExpression mode /// and we produce one of these. OptionalDiagnostic CCEDiag(SourceLocation Loc, diag::kind DiagId = diag::note_invalid_subexpr_in_const_expr, unsigned ExtraNotes = 0) { // Don't override a previous diagnostic. Don't bother collecting // diagnostics if we're evaluating for overflow. if (!EvalStatus.Diag || !EvalStatus.Diag->empty()) { HasActiveDiagnostic = false; return OptionalDiagnostic(); } return Diag(Loc, DiagId, ExtraNotes, true); } OptionalDiagnostic CCEDiag(const Expr *E, diag::kind DiagId = diag::note_invalid_subexpr_in_const_expr, unsigned ExtraNotes = 0) { return CCEDiag(E->getExprLoc(), DiagId, ExtraNotes); } /// Add a note to a prior diagnostic. OptionalDiagnostic Note(SourceLocation Loc, diag::kind DiagId) { if (!HasActiveDiagnostic) return OptionalDiagnostic(); return OptionalDiagnostic(&addDiag(Loc, DiagId)); } /// Add a stack of notes to a prior diagnostic. void addNotes(ArrayRef Diags) { if (HasActiveDiagnostic) { EvalStatus.Diag->insert(EvalStatus.Diag->end(), Diags.begin(), Diags.end()); } } /// Should we continue evaluation after encountering a side-effect that we /// couldn't model? bool keepEvaluatingAfterSideEffect() { switch (EvalMode) { case EM_IgnoreSideEffects: return true; case EM_ConstantExpression: case EM_ConstantExpressionUnevaluated: case EM_ConstantFold: // By default, assume any side effect might be valid in some other // evaluation of this expression from a different context. return checkingPotentialConstantExpression() || checkingForUndefinedBehavior(); } llvm_unreachable("Missed EvalMode case"); } /// Note that we have had a side-effect, and determine whether we should /// keep evaluating. bool noteSideEffect() { EvalStatus.HasSideEffects = true; return keepEvaluatingAfterSideEffect(); } /// Should we continue evaluation after encountering undefined behavior? bool keepEvaluatingAfterUndefinedBehavior() { switch (EvalMode) { case EM_IgnoreSideEffects: case EM_ConstantFold: return true; case EM_ConstantExpression: case EM_ConstantExpressionUnevaluated: return checkingForUndefinedBehavior(); } llvm_unreachable("Missed EvalMode case"); } /// Note that we hit something that was technically undefined behavior, but /// that we can evaluate past it (such as signed overflow or floating-point /// division by zero.) bool noteUndefinedBehavior() { EvalStatus.HasUndefinedBehavior = true; return keepEvaluatingAfterUndefinedBehavior(); } /// Should we continue evaluation as much as possible after encountering a /// construct which can't be reduced to a value? bool keepEvaluatingAfterFailure() { if (!StepsLeft) return false; switch (EvalMode) { case EM_ConstantExpression: case EM_ConstantExpressionUnevaluated: case EM_ConstantFold: case EM_IgnoreSideEffects: return checkingPotentialConstantExpression() || checkingForUndefinedBehavior(); } llvm_unreachable("Missed EvalMode case"); } /// Notes that we failed to evaluate an expression that other expressions /// directly depend on, and determine if we should keep evaluating. This /// should only be called if we actually intend to keep evaluating. /// /// Call noteSideEffect() instead if we may be able to ignore the value that /// we failed to evaluate, e.g. if we failed to evaluate Foo() in: /// /// (Foo(), 1) // use noteSideEffect /// (Foo() || true) // use noteSideEffect /// Foo() + 1 // use noteFailure LLVM_NODISCARD bool noteFailure() { // Failure when evaluating some expression often means there is some // subexpression whose evaluation was skipped. Therefore, (because we // don't track whether we skipped an expression when unwinding after an // evaluation failure) every evaluation failure that bubbles up from a // subexpression implies that a side-effect has potentially happened. We // skip setting the HasSideEffects flag to true until we decide to // continue evaluating after that point, which happens here. bool KeepGoing = keepEvaluatingAfterFailure(); EvalStatus.HasSideEffects |= KeepGoing; return KeepGoing; } class ArrayInitLoopIndex { EvalInfo &Info; uint64_t OuterIndex; public: ArrayInitLoopIndex(EvalInfo &Info) : Info(Info), OuterIndex(Info.ArrayInitIndex) { Info.ArrayInitIndex = 0; } ~ArrayInitLoopIndex() { Info.ArrayInitIndex = OuterIndex; } operator uint64_t&() { return Info.ArrayInitIndex; } }; }; /// Object used to treat all foldable expressions as constant expressions. struct FoldConstant { EvalInfo &Info; bool Enabled; bool HadNoPriorDiags; EvalInfo::EvaluationMode OldMode; explicit FoldConstant(EvalInfo &Info, bool Enabled) : Info(Info), Enabled(Enabled), HadNoPriorDiags(Info.EvalStatus.Diag && Info.EvalStatus.Diag->empty() && !Info.EvalStatus.HasSideEffects), OldMode(Info.EvalMode) { if (Enabled) Info.EvalMode = EvalInfo::EM_ConstantFold; } void keepDiagnostics() { Enabled = false; } ~FoldConstant() { if (Enabled && HadNoPriorDiags && !Info.EvalStatus.Diag->empty() && !Info.EvalStatus.HasSideEffects) Info.EvalStatus.Diag->clear(); Info.EvalMode = OldMode; } }; /// RAII object used to set the current evaluation mode to ignore /// side-effects. struct IgnoreSideEffectsRAII { EvalInfo &Info; EvalInfo::EvaluationMode OldMode; explicit IgnoreSideEffectsRAII(EvalInfo &Info) : Info(Info), OldMode(Info.EvalMode) { Info.EvalMode = EvalInfo::EM_IgnoreSideEffects; } ~IgnoreSideEffectsRAII() { Info.EvalMode = OldMode; } }; /// RAII object used to optionally suppress diagnostics and side-effects from /// a speculative evaluation. class SpeculativeEvaluationRAII { EvalInfo *Info = nullptr; Expr::EvalStatus OldStatus; unsigned OldSpeculativeEvaluationDepth; void moveFromAndCancel(SpeculativeEvaluationRAII &&Other) { Info = Other.Info; OldStatus = Other.OldStatus; OldSpeculativeEvaluationDepth = Other.OldSpeculativeEvaluationDepth; Other.Info = nullptr; } void maybeRestoreState() { if (!Info) return; Info->EvalStatus = OldStatus; Info->SpeculativeEvaluationDepth = OldSpeculativeEvaluationDepth; } public: SpeculativeEvaluationRAII() = default; SpeculativeEvaluationRAII( EvalInfo &Info, SmallVectorImpl *NewDiag = nullptr) : Info(&Info), OldStatus(Info.EvalStatus), OldSpeculativeEvaluationDepth(Info.SpeculativeEvaluationDepth) { Info.EvalStatus.Diag = NewDiag; Info.SpeculativeEvaluationDepth = Info.CallStackDepth + 1; } SpeculativeEvaluationRAII(const SpeculativeEvaluationRAII &Other) = delete; SpeculativeEvaluationRAII(SpeculativeEvaluationRAII &&Other) { moveFromAndCancel(std::move(Other)); } SpeculativeEvaluationRAII &operator=(SpeculativeEvaluationRAII &&Other) { maybeRestoreState(); moveFromAndCancel(std::move(Other)); return *this; } ~SpeculativeEvaluationRAII() { maybeRestoreState(); } }; /// RAII object wrapping a full-expression or block scope, and handling /// the ending of the lifetime of temporaries created within it. template class ScopeRAII { EvalInfo &Info; unsigned OldStackSize; public: ScopeRAII(EvalInfo &Info) : Info(Info), OldStackSize(Info.CleanupStack.size()) { // Push a new temporary version. This is needed to distinguish between // temporaries created in different iterations of a loop. Info.CurrentCall->pushTempVersion(); } ~ScopeRAII() { // Body moved to a static method to encourage the compiler to inline away // instances of this class. cleanup(Info, OldStackSize); Info.CurrentCall->popTempVersion(); } private: static void cleanup(EvalInfo &Info, unsigned OldStackSize) { unsigned NewEnd = OldStackSize; for (unsigned I = OldStackSize, N = Info.CleanupStack.size(); I != N; ++I) { if (IsFullExpression && Info.CleanupStack[I].isLifetimeExtended()) { // Full-expression cleanup of a lifetime-extended temporary: nothing // to do, just move this cleanup to the right place in the stack. std::swap(Info.CleanupStack[I], Info.CleanupStack[NewEnd]); ++NewEnd; } else { // End the lifetime of the object. Info.CleanupStack[I].endLifetime(); } } Info.CleanupStack.erase(Info.CleanupStack.begin() + NewEnd, Info.CleanupStack.end()); } }; typedef ScopeRAII BlockScopeRAII; typedef ScopeRAII FullExpressionRAII; } bool SubobjectDesignator::checkSubobject(EvalInfo &Info, const Expr *E, CheckSubobjectKind CSK) { if (Invalid) return false; if (isOnePastTheEnd()) { Info.CCEDiag(E, diag::note_constexpr_past_end_subobject) << CSK; setInvalid(); return false; } // Note, we do not diagnose if isMostDerivedAnUnsizedArray(), because there // must actually be at least one array element; even a VLA cannot have a // bound of zero. And if our index is nonzero, we already had a CCEDiag. return true; } void SubobjectDesignator::diagnoseUnsizedArrayPointerArithmetic(EvalInfo &Info, const Expr *E) { Info.CCEDiag(E, diag::note_constexpr_unsized_array_indexed); // Do not set the designator as invalid: we can represent this situation, // and correct handling of __builtin_object_size requires us to do so. } void SubobjectDesignator::diagnosePointerArithmetic(EvalInfo &Info, const Expr *E, const APSInt &N) { // If we're complaining, we must be able to statically determine the size of // the most derived array. if (MostDerivedPathLength == Entries.size() && MostDerivedIsArrayElement) Info.CCEDiag(E, diag::note_constexpr_array_index) << N << /*array*/ 0 << static_cast(getMostDerivedArraySize()); else Info.CCEDiag(E, diag::note_constexpr_array_index) << N << /*non-array*/ 1; setInvalid(); } CallStackFrame::CallStackFrame(EvalInfo &Info, SourceLocation CallLoc, const FunctionDecl *Callee, const LValue *This, APValue *Arguments) : Info(Info), Caller(Info.CurrentCall), Callee(Callee), This(This), Arguments(Arguments), CallLoc(CallLoc), Index(Info.NextCallIndex++) { Info.CurrentCall = this; ++Info.CallStackDepth; } CallStackFrame::~CallStackFrame() { assert(Info.CurrentCall == this && "calls retired out of order"); --Info.CallStackDepth; Info.CurrentCall = Caller; } APValue &CallStackFrame::createTemporary(const void *Key, bool IsLifetimeExtended) { unsigned Version = Info.CurrentCall->getTempVersion(); APValue &Result = Temporaries[MapKeyTy(Key, Version)]; assert(Result.isAbsent() && "temporary created multiple times"); Info.CleanupStack.push_back(Cleanup(&Result, IsLifetimeExtended)); return Result; } static void describeCall(CallStackFrame *Frame, raw_ostream &Out); void EvalInfo::addCallStack(unsigned Limit) { // Determine which calls to skip, if any. unsigned ActiveCalls = CallStackDepth - 1; unsigned SkipStart = ActiveCalls, SkipEnd = SkipStart; if (Limit && Limit < ActiveCalls) { SkipStart = Limit / 2 + Limit % 2; SkipEnd = ActiveCalls - Limit / 2; } // Walk the call stack and add the diagnostics. unsigned CallIdx = 0; for (CallStackFrame *Frame = CurrentCall; Frame != &BottomFrame; Frame = Frame->Caller, ++CallIdx) { // Skip this call? if (CallIdx >= SkipStart && CallIdx < SkipEnd) { if (CallIdx == SkipStart) { // Note that we're skipping calls. addDiag(Frame->CallLoc, diag::note_constexpr_calls_suppressed) << unsigned(ActiveCalls - Limit); } continue; } // Use a different note for an inheriting constructor, because from the // user's perspective it's not really a function at all. if (auto *CD = dyn_cast_or_null(Frame->Callee)) { if (CD->isInheritingConstructor()) { addDiag(Frame->CallLoc, diag::note_constexpr_inherited_ctor_call_here) << CD->getParent(); continue; } } SmallVector Buffer; llvm::raw_svector_ostream Out(Buffer); describeCall(Frame, Out); addDiag(Frame->CallLoc, diag::note_constexpr_call_here) << Out.str(); } } /// Kinds of access we can perform on an object, for diagnostics. Note that /// we consider a member function call to be a kind of access, even though /// it is not formally an access of the object, because it has (largely) the /// same set of semantic restrictions. enum AccessKinds { AK_Read, AK_Assign, AK_Increment, AK_Decrement, AK_MemberCall, AK_DynamicCast, AK_TypeId, }; static bool isModification(AccessKinds AK) { switch (AK) { case AK_Read: case AK_MemberCall: case AK_DynamicCast: case AK_TypeId: return false; case AK_Assign: case AK_Increment: case AK_Decrement: return true; } llvm_unreachable("unknown access kind"); } /// Is this an access per the C++ definition? static bool isFormalAccess(AccessKinds AK) { return AK == AK_Read || isModification(AK); } namespace { struct ComplexValue { private: bool IsInt; public: APSInt IntReal, IntImag; APFloat FloatReal, FloatImag; ComplexValue() : FloatReal(APFloat::Bogus()), FloatImag(APFloat::Bogus()) {} void makeComplexFloat() { IsInt = false; } bool isComplexFloat() const { return !IsInt; } APFloat &getComplexFloatReal() { return FloatReal; } APFloat &getComplexFloatImag() { return FloatImag; } void makeComplexInt() { IsInt = true; } bool isComplexInt() const { return IsInt; } APSInt &getComplexIntReal() { return IntReal; } APSInt &getComplexIntImag() { return IntImag; } void moveInto(APValue &v) const { if (isComplexFloat()) v = APValue(FloatReal, FloatImag); else v = APValue(IntReal, IntImag); } void setFrom(const APValue &v) { assert(v.isComplexFloat() || v.isComplexInt()); if (v.isComplexFloat()) { makeComplexFloat(); FloatReal = v.getComplexFloatReal(); FloatImag = v.getComplexFloatImag(); } else { makeComplexInt(); IntReal = v.getComplexIntReal(); IntImag = v.getComplexIntImag(); } } }; struct LValue { APValue::LValueBase Base; CharUnits Offset; SubobjectDesignator Designator; bool IsNullPtr : 1; bool InvalidBase : 1; const APValue::LValueBase getLValueBase() const { return Base; } CharUnits &getLValueOffset() { return Offset; } const CharUnits &getLValueOffset() const { return Offset; } SubobjectDesignator &getLValueDesignator() { return Designator; } const SubobjectDesignator &getLValueDesignator() const { return Designator;} bool isNullPointer() const { return IsNullPtr;} unsigned getLValueCallIndex() const { return Base.getCallIndex(); } unsigned getLValueVersion() const { return Base.getVersion(); } void moveInto(APValue &V) const { if (Designator.Invalid) V = APValue(Base, Offset, APValue::NoLValuePath(), IsNullPtr); else { assert(!InvalidBase && "APValues can't handle invalid LValue bases"); V = APValue(Base, Offset, Designator.Entries, Designator.IsOnePastTheEnd, IsNullPtr); } } void setFrom(ASTContext &Ctx, const APValue &V) { assert(V.isLValue() && "Setting LValue from a non-LValue?"); Base = V.getLValueBase(); Offset = V.getLValueOffset(); InvalidBase = false; Designator = SubobjectDesignator(Ctx, V); IsNullPtr = V.isNullPointer(); } void set(APValue::LValueBase B, bool BInvalid = false) { #ifndef NDEBUG // We only allow a few types of invalid bases. Enforce that here. if (BInvalid) { const auto *E = B.get(); assert((isa(E) || tryUnwrapAllocSizeCall(E)) && "Unexpected type of invalid base"); } #endif Base = B; Offset = CharUnits::fromQuantity(0); InvalidBase = BInvalid; Designator = SubobjectDesignator(getType(B)); IsNullPtr = false; } void setNull(QualType PointerTy, uint64_t TargetVal) { Base = (Expr *)nullptr; Offset = CharUnits::fromQuantity(TargetVal); InvalidBase = false; Designator = SubobjectDesignator(PointerTy->getPointeeType()); IsNullPtr = true; } void setInvalid(APValue::LValueBase B, unsigned I = 0) { set(B, true); } private: // Check that this LValue is not based on a null pointer. If it is, produce // a diagnostic and mark the designator as invalid. template bool checkNullPointerDiagnosingWith(const GenDiagType &GenDiag) { if (Designator.Invalid) return false; if (IsNullPtr) { GenDiag(); Designator.setInvalid(); return false; } return true; } public: bool checkNullPointer(EvalInfo &Info, const Expr *E, CheckSubobjectKind CSK) { return checkNullPointerDiagnosingWith([&Info, E, CSK] { Info.CCEDiag(E, diag::note_constexpr_null_subobject) << CSK; }); } bool checkNullPointerForFoldAccess(EvalInfo &Info, const Expr *E, AccessKinds AK) { return checkNullPointerDiagnosingWith([&Info, E, AK] { Info.FFDiag(E, diag::note_constexpr_access_null) << AK; }); } // Check this LValue refers to an object. If not, set the designator to be // invalid and emit a diagnostic. bool checkSubobject(EvalInfo &Info, const Expr *E, CheckSubobjectKind CSK) { return (CSK == CSK_ArrayToPointer || checkNullPointer(Info, E, CSK)) && Designator.checkSubobject(Info, E, CSK); } void addDecl(EvalInfo &Info, const Expr *E, const Decl *D, bool Virtual = false) { if (checkSubobject(Info, E, isa(D) ? CSK_Field : CSK_Base)) Designator.addDeclUnchecked(D, Virtual); } void addUnsizedArray(EvalInfo &Info, const Expr *E, QualType ElemTy) { if (!Designator.Entries.empty()) { Info.CCEDiag(E, diag::note_constexpr_unsupported_unsized_array); Designator.setInvalid(); return; } if (checkSubobject(Info, E, CSK_ArrayToPointer)) { assert(getType(Base)->isPointerType() || getType(Base)->isArrayType()); Designator.FirstEntryIsAnUnsizedArray = true; Designator.addUnsizedArrayUnchecked(ElemTy); } } void addArray(EvalInfo &Info, const Expr *E, const ConstantArrayType *CAT) { if (checkSubobject(Info, E, CSK_ArrayToPointer)) Designator.addArrayUnchecked(CAT); } void addComplex(EvalInfo &Info, const Expr *E, QualType EltTy, bool Imag) { if (checkSubobject(Info, E, Imag ? CSK_Imag : CSK_Real)) Designator.addComplexUnchecked(EltTy, Imag); } void clearIsNullPointer() { IsNullPtr = false; } void adjustOffsetAndIndex(EvalInfo &Info, const Expr *E, const APSInt &Index, CharUnits ElementSize) { // An index of 0 has no effect. (In C, adding 0 to a null pointer is UB, // but we're not required to diagnose it and it's valid in C++.) if (!Index) return; // Compute the new offset in the appropriate width, wrapping at 64 bits. // FIXME: When compiling for a 32-bit target, we should use 32-bit // offsets. uint64_t Offset64 = Offset.getQuantity(); uint64_t ElemSize64 = ElementSize.getQuantity(); uint64_t Index64 = Index.extOrTrunc(64).getZExtValue(); Offset = CharUnits::fromQuantity(Offset64 + ElemSize64 * Index64); if (checkNullPointer(Info, E, CSK_ArrayIndex)) Designator.adjustIndex(Info, E, Index); clearIsNullPointer(); } void adjustOffset(CharUnits N) { Offset += N; if (N.getQuantity()) clearIsNullPointer(); } }; struct MemberPtr { MemberPtr() {} explicit MemberPtr(const ValueDecl *Decl) : DeclAndIsDerivedMember(Decl, false), Path() {} /// The member or (direct or indirect) field referred to by this member /// pointer, or 0 if this is a null member pointer. const ValueDecl *getDecl() const { return DeclAndIsDerivedMember.getPointer(); } /// Is this actually a member of some type derived from the relevant class? bool isDerivedMember() const { return DeclAndIsDerivedMember.getInt(); } /// Get the class which the declaration actually lives in. const CXXRecordDecl *getContainingRecord() const { return cast( DeclAndIsDerivedMember.getPointer()->getDeclContext()); } void moveInto(APValue &V) const { V = APValue(getDecl(), isDerivedMember(), Path); } void setFrom(const APValue &V) { assert(V.isMemberPointer()); DeclAndIsDerivedMember.setPointer(V.getMemberPointerDecl()); DeclAndIsDerivedMember.setInt(V.isMemberPointerToDerivedMember()); Path.clear(); ArrayRef P = V.getMemberPointerPath(); Path.insert(Path.end(), P.begin(), P.end()); } /// DeclAndIsDerivedMember - The member declaration, and a flag indicating /// whether the member is a member of some class derived from the class type /// of the member pointer. llvm::PointerIntPair DeclAndIsDerivedMember; /// Path - The path of base/derived classes from the member declaration's /// class (exclusive) to the class type of the member pointer (inclusive). SmallVector Path; /// Perform a cast towards the class of the Decl (either up or down the /// hierarchy). bool castBack(const CXXRecordDecl *Class) { assert(!Path.empty()); const CXXRecordDecl *Expected; if (Path.size() >= 2) Expected = Path[Path.size() - 2]; else Expected = getContainingRecord(); if (Expected->getCanonicalDecl() != Class->getCanonicalDecl()) { // C++11 [expr.static.cast]p12: In a conversion from (D::*) to (B::*), // if B does not contain the original member and is not a base or // derived class of the class containing the original member, the result // of the cast is undefined. // C++11 [conv.mem]p2 does not cover this case for a cast from (B::*) to // (D::*). We consider that to be a language defect. return false; } Path.pop_back(); return true; } /// Perform a base-to-derived member pointer cast. bool castToDerived(const CXXRecordDecl *Derived) { if (!getDecl()) return true; if (!isDerivedMember()) { Path.push_back(Derived); return true; } if (!castBack(Derived)) return false; if (Path.empty()) DeclAndIsDerivedMember.setInt(false); return true; } /// Perform a derived-to-base member pointer cast. bool castToBase(const CXXRecordDecl *Base) { if (!getDecl()) return true; if (Path.empty()) DeclAndIsDerivedMember.setInt(true); if (isDerivedMember()) { Path.push_back(Base); return true; } return castBack(Base); } }; /// Compare two member pointers, which are assumed to be of the same type. static bool operator==(const MemberPtr &LHS, const MemberPtr &RHS) { if (!LHS.getDecl() || !RHS.getDecl()) return !LHS.getDecl() && !RHS.getDecl(); if (LHS.getDecl()->getCanonicalDecl() != RHS.getDecl()->getCanonicalDecl()) return false; return LHS.Path == RHS.Path; } } static bool Evaluate(APValue &Result, EvalInfo &Info, const Expr *E); static bool EvaluateInPlace(APValue &Result, EvalInfo &Info, const LValue &This, const Expr *E, bool AllowNonLiteralTypes = false); static bool EvaluateLValue(const Expr *E, LValue &Result, EvalInfo &Info, bool InvalidBaseOK = false); static bool EvaluatePointer(const Expr *E, LValue &Result, EvalInfo &Info, bool InvalidBaseOK = false); static bool EvaluateMemberPointer(const Expr *E, MemberPtr &Result, EvalInfo &Info); static bool EvaluateTemporary(const Expr *E, LValue &Result, EvalInfo &Info); static bool EvaluateInteger(const Expr *E, APSInt &Result, EvalInfo &Info); static bool EvaluateIntegerOrLValue(const Expr *E, APValue &Result, EvalInfo &Info); static bool EvaluateFloat(const Expr *E, APFloat &Result, EvalInfo &Info); static bool EvaluateComplex(const Expr *E, ComplexValue &Res, EvalInfo &Info); static bool EvaluateAtomic(const Expr *E, const LValue *This, APValue &Result, EvalInfo &Info); static bool EvaluateAsRValue(EvalInfo &Info, const Expr *E, APValue &Result); /// Evaluate an integer or fixed point expression into an APResult. static bool EvaluateFixedPointOrInteger(const Expr *E, APFixedPoint &Result, EvalInfo &Info); /// Evaluate only a fixed point expression into an APResult. static bool EvaluateFixedPoint(const Expr *E, APFixedPoint &Result, EvalInfo &Info); //===----------------------------------------------------------------------===// // Misc utilities //===----------------------------------------------------------------------===// /// A helper function to create a temporary and set an LValue. template static APValue &createTemporary(const KeyTy *Key, bool IsLifetimeExtended, LValue &LV, CallStackFrame &Frame) { LV.set({Key, Frame.Info.CurrentCall->Index, Frame.Info.CurrentCall->getTempVersion()}); return Frame.createTemporary(Key, IsLifetimeExtended); } /// Negate an APSInt in place, converting it to a signed form if necessary, and /// preserving its value (by extending by up to one bit as needed). static void negateAsSigned(APSInt &Int) { if (Int.isUnsigned() || Int.isMinSignedValue()) { Int = Int.extend(Int.getBitWidth() + 1); Int.setIsSigned(true); } Int = -Int; } /// Produce a string describing the given constexpr call. static void describeCall(CallStackFrame *Frame, raw_ostream &Out) { unsigned ArgIndex = 0; bool IsMemberCall = isa(Frame->Callee) && !isa(Frame->Callee) && cast(Frame->Callee)->isInstance(); if (!IsMemberCall) Out << *Frame->Callee << '('; if (Frame->This && IsMemberCall) { APValue Val; Frame->This->moveInto(Val); Val.printPretty(Out, Frame->Info.Ctx, Frame->This->Designator.MostDerivedType); // FIXME: Add parens around Val if needed. Out << "->" << *Frame->Callee << '('; IsMemberCall = false; } for (FunctionDecl::param_const_iterator I = Frame->Callee->param_begin(), E = Frame->Callee->param_end(); I != E; ++I, ++ArgIndex) { if (ArgIndex > (unsigned)IsMemberCall) Out << ", "; const ParmVarDecl *Param = *I; const APValue &Arg = Frame->Arguments[ArgIndex]; Arg.printPretty(Out, Frame->Info.Ctx, Param->getType()); if (ArgIndex == 0 && IsMemberCall) Out << "->" << *Frame->Callee << '('; } Out << ')'; } /// Evaluate an expression to see if it had side-effects, and discard its /// result. /// \return \c true if the caller should keep evaluating. static bool EvaluateIgnoredValue(EvalInfo &Info, const Expr *E) { APValue Scratch; if (!Evaluate(Scratch, Info, E)) // We don't need the value, but we might have skipped a side effect here. return Info.noteSideEffect(); return true; } /// Should this call expression be treated as a string literal? static bool IsStringLiteralCall(const CallExpr *E) { unsigned Builtin = E->getBuiltinCallee(); return (Builtin == Builtin::BI__builtin___CFStringMakeConstantString || Builtin == Builtin::BI__builtin___NSStringMakeConstantString); } static bool IsGlobalLValue(APValue::LValueBase B) { // C++11 [expr.const]p3 An address constant expression is a prvalue core // constant expression of pointer type that evaluates to... // ... a null pointer value, or a prvalue core constant expression of type // std::nullptr_t. if (!B) return true; if (const ValueDecl *D = B.dyn_cast()) { // ... the address of an object with static storage duration, if (const VarDecl *VD = dyn_cast(D)) return VD->hasGlobalStorage(); // ... the address of a function, return isa(D); } if (B.is()) return true; const Expr *E = B.get(); switch (E->getStmtClass()) { default: return false; case Expr::CompoundLiteralExprClass: { const CompoundLiteralExpr *CLE = cast(E); return CLE->isFileScope() && CLE->isLValue(); } case Expr::MaterializeTemporaryExprClass: // A materialized temporary might have been lifetime-extended to static // storage duration. return cast(E)->getStorageDuration() == SD_Static; // A string literal has static storage duration. case Expr::StringLiteralClass: case Expr::PredefinedExprClass: case Expr::ObjCStringLiteralClass: case Expr::ObjCEncodeExprClass: case Expr::CXXUuidofExprClass: return true; case Expr::ObjCBoxedExprClass: return cast(E)->isExpressibleAsConstantInitializer(); case Expr::CallExprClass: return IsStringLiteralCall(cast(E)); // For GCC compatibility, &&label has static storage duration. case Expr::AddrLabelExprClass: return true; // A Block literal expression may be used as the initialization value for // Block variables at global or local static scope. case Expr::BlockExprClass: return !cast(E)->getBlockDecl()->hasCaptures(); case Expr::ImplicitValueInitExprClass: // FIXME: // We can never form an lvalue with an implicit value initialization as its // base through expression evaluation, so these only appear in one case: the // implicit variable declaration we invent when checking whether a constexpr // constructor can produce a constant expression. We must assume that such // an expression might be a global lvalue. return true; } } static const ValueDecl *GetLValueBaseDecl(const LValue &LVal) { return LVal.Base.dyn_cast(); } static bool IsLiteralLValue(const LValue &Value) { if (Value.getLValueCallIndex()) return false; const Expr *E = Value.Base.dyn_cast(); return E && !isa(E); } static bool IsWeakLValue(const LValue &Value) { const ValueDecl *Decl = GetLValueBaseDecl(Value); return Decl && Decl->isWeak(); } static bool isZeroSized(const LValue &Value) { const ValueDecl *Decl = GetLValueBaseDecl(Value); if (Decl && isa(Decl)) { QualType Ty = Decl->getType(); if (Ty->isArrayType()) return Ty->isIncompleteType() || Decl->getASTContext().getTypeSize(Ty) == 0; } return false; } static bool HasSameBase(const LValue &A, const LValue &B) { if (!A.getLValueBase()) return !B.getLValueBase(); if (!B.getLValueBase()) return false; if (A.getLValueBase().getOpaqueValue() != B.getLValueBase().getOpaqueValue()) { const Decl *ADecl = GetLValueBaseDecl(A); if (!ADecl) return false; const Decl *BDecl = GetLValueBaseDecl(B); if (!BDecl || ADecl->getCanonicalDecl() != BDecl->getCanonicalDecl()) return false; } return IsGlobalLValue(A.getLValueBase()) || (A.getLValueCallIndex() == B.getLValueCallIndex() && A.getLValueVersion() == B.getLValueVersion()); } static void NoteLValueLocation(EvalInfo &Info, APValue::LValueBase Base) { assert(Base && "no location for a null lvalue"); const ValueDecl *VD = Base.dyn_cast(); if (VD) Info.Note(VD->getLocation(), diag::note_declared_at); else if (const Expr *E = Base.dyn_cast()) Info.Note(E->getExprLoc(), diag::note_constexpr_temporary_here); // We have no information to show for a typeid(T) object. } /// Check that this reference or pointer core constant expression is a valid /// value for an address or reference constant expression. Return true if we /// can fold this expression, whether or not it's a constant expression. static bool CheckLValueConstantExpression(EvalInfo &Info, SourceLocation Loc, QualType Type, const LValue &LVal, Expr::ConstExprUsage Usage) { bool IsReferenceType = Type->isReferenceType(); APValue::LValueBase Base = LVal.getLValueBase(); const SubobjectDesignator &Designator = LVal.getLValueDesignator(); // Check that the object is a global. Note that the fake 'this' object we // manufacture when checking potential constant expressions is conservatively // assumed to be global here. if (!IsGlobalLValue(Base)) { if (Info.getLangOpts().CPlusPlus11) { const ValueDecl *VD = Base.dyn_cast(); Info.FFDiag(Loc, diag::note_constexpr_non_global, 1) << IsReferenceType << !Designator.Entries.empty() << !!VD << VD; NoteLValueLocation(Info, Base); } else { Info.FFDiag(Loc); } // Don't allow references to temporaries to escape. return false; } assert((Info.checkingPotentialConstantExpression() || LVal.getLValueCallIndex() == 0) && "have call index for global lvalue"); if (const ValueDecl *VD = Base.dyn_cast()) { if (const VarDecl *Var = dyn_cast(VD)) { // Check if this is a thread-local variable. if (Var->getTLSKind()) return false; // A dllimport variable never acts like a constant. if (Usage == Expr::EvaluateForCodeGen && Var->hasAttr()) return false; } if (const auto *FD = dyn_cast(VD)) { // __declspec(dllimport) must be handled very carefully: // We must never initialize an expression with the thunk in C++. // Doing otherwise would allow the same id-expression to yield // different addresses for the same function in different translation // units. However, this means that we must dynamically initialize the // expression with the contents of the import address table at runtime. // // The C language has no notion of ODR; furthermore, it has no notion of // dynamic initialization. This means that we are permitted to // perform initialization with the address of the thunk. if (Info.getLangOpts().CPlusPlus && Usage == Expr::EvaluateForCodeGen && FD->hasAttr()) return false; } } // Allow address constant expressions to be past-the-end pointers. This is // an extension: the standard requires them to point to an object. if (!IsReferenceType) return true; // A reference constant expression must refer to an object. if (!Base) { // FIXME: diagnostic Info.CCEDiag(Loc); return true; } // Does this refer one past the end of some object? if (!Designator.Invalid && Designator.isOnePastTheEnd()) { const ValueDecl *VD = Base.dyn_cast(); Info.FFDiag(Loc, diag::note_constexpr_past_end, 1) << !Designator.Entries.empty() << !!VD << VD; NoteLValueLocation(Info, Base); } return true; } /// Member pointers are constant expressions unless they point to a /// non-virtual dllimport member function. static bool CheckMemberPointerConstantExpression(EvalInfo &Info, SourceLocation Loc, QualType Type, const APValue &Value, Expr::ConstExprUsage Usage) { const ValueDecl *Member = Value.getMemberPointerDecl(); const auto *FD = dyn_cast_or_null(Member); if (!FD) return true; return Usage == Expr::EvaluateForMangling || FD->isVirtual() || !FD->hasAttr(); } /// Check that this core constant expression is of literal type, and if not, /// produce an appropriate diagnostic. static bool CheckLiteralType(EvalInfo &Info, const Expr *E, const LValue *This = nullptr) { if (!E->isRValue() || E->getType()->isLiteralType(Info.Ctx)) return true; // C++1y: A constant initializer for an object o [...] may also invoke // constexpr constructors for o and its subobjects even if those objects // are of non-literal class types. // // C++11 missed this detail for aggregates, so classes like this: // struct foo_t { union { int i; volatile int j; } u; }; // are not (obviously) initializable like so: // __attribute__((__require_constant_initialization__)) // static const foo_t x = {{0}}; // because "i" is a subobject with non-literal initialization (due to the // volatile member of the union). See: // http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_active.html#1677 // Therefore, we use the C++1y behavior. if (This && Info.EvaluatingDecl == This->getLValueBase()) return true; // Prvalue constant expressions must be of literal types. if (Info.getLangOpts().CPlusPlus11) Info.FFDiag(E, diag::note_constexpr_nonliteral) << E->getType(); else Info.FFDiag(E, diag::note_invalid_subexpr_in_const_expr); return false; } /// Check that this core constant expression value is a valid value for a /// constant expression. If not, report an appropriate diagnostic. Does not /// check that the expression is of literal type. static bool CheckConstantExpression(EvalInfo &Info, SourceLocation DiagLoc, QualType Type, const APValue &Value, Expr::ConstExprUsage Usage = Expr::EvaluateForCodeGen, SourceLocation SubobjectLoc = SourceLocation()) { if (!Value.hasValue()) { Info.FFDiag(DiagLoc, diag::note_constexpr_uninitialized) << true << Type; if (SubobjectLoc.isValid()) Info.Note(SubobjectLoc, diag::note_constexpr_subobject_declared_here); return false; } // We allow _Atomic(T) to be initialized from anything that T can be // initialized from. if (const AtomicType *AT = Type->getAs()) Type = AT->getValueType(); // Core issue 1454: For a literal constant expression of array or class type, // each subobject of its value shall have been initialized by a constant // expression. if (Value.isArray()) { QualType EltTy = Type->castAsArrayTypeUnsafe()->getElementType(); for (unsigned I = 0, N = Value.getArrayInitializedElts(); I != N; ++I) { if (!CheckConstantExpression(Info, DiagLoc, EltTy, Value.getArrayInitializedElt(I), Usage, SubobjectLoc)) return false; } if (!Value.hasArrayFiller()) return true; return CheckConstantExpression(Info, DiagLoc, EltTy, Value.getArrayFiller(), Usage, SubobjectLoc); } if (Value.isUnion() && Value.getUnionField()) { return CheckConstantExpression(Info, DiagLoc, Value.getUnionField()->getType(), Value.getUnionValue(), Usage, Value.getUnionField()->getLocation()); } if (Value.isStruct()) { RecordDecl *RD = Type->castAs()->getDecl(); if (const CXXRecordDecl *CD = dyn_cast(RD)) { unsigned BaseIndex = 0; for (const CXXBaseSpecifier &BS : CD->bases()) { if (!CheckConstantExpression(Info, DiagLoc, BS.getType(), Value.getStructBase(BaseIndex), Usage, BS.getBeginLoc())) return false; ++BaseIndex; } } for (const auto *I : RD->fields()) { if (I->isUnnamedBitfield()) continue; if (!CheckConstantExpression(Info, DiagLoc, I->getType(), Value.getStructField(I->getFieldIndex()), Usage, I->getLocation())) return false; } } if (Value.isLValue()) { LValue LVal; LVal.setFrom(Info.Ctx, Value); return CheckLValueConstantExpression(Info, DiagLoc, Type, LVal, Usage); } if (Value.isMemberPointer()) return CheckMemberPointerConstantExpression(Info, DiagLoc, Type, Value, Usage); // Everything else is fine. return true; } static bool EvalPointerValueAsBool(const APValue &Value, bool &Result) { // A null base expression indicates a null pointer. These are always // evaluatable, and they are false unless the offset is zero. if (!Value.getLValueBase()) { Result = !Value.getLValueOffset().isZero(); return true; } // We have a non-null base. These are generally known to be true, but if it's // a weak declaration it can be null at runtime. Result = true; const ValueDecl *Decl = Value.getLValueBase().dyn_cast(); return !Decl || !Decl->isWeak(); } static bool HandleConversionToBool(const APValue &Val, bool &Result) { switch (Val.getKind()) { case APValue::None: case APValue::Indeterminate: return false; case APValue::Int: Result = Val.getInt().getBoolValue(); return true; case APValue::FixedPoint: Result = Val.getFixedPoint().getBoolValue(); return true; case APValue::Float: Result = !Val.getFloat().isZero(); return true; case APValue::ComplexInt: Result = Val.getComplexIntReal().getBoolValue() || Val.getComplexIntImag().getBoolValue(); return true; case APValue::ComplexFloat: Result = !Val.getComplexFloatReal().isZero() || !Val.getComplexFloatImag().isZero(); return true; case APValue::LValue: return EvalPointerValueAsBool(Val, Result); case APValue::MemberPointer: Result = Val.getMemberPointerDecl(); return true; case APValue::Vector: case APValue::Array: case APValue::Struct: case APValue::Union: case APValue::AddrLabelDiff: return false; } llvm_unreachable("unknown APValue kind"); } static bool EvaluateAsBooleanCondition(const Expr *E, bool &Result, EvalInfo &Info) { assert(E->isRValue() && "missing lvalue-to-rvalue conv in bool condition"); APValue Val; if (!Evaluate(Val, Info, E)) return false; return HandleConversionToBool(Val, Result); } template static bool HandleOverflow(EvalInfo &Info, const Expr *E, const T &SrcValue, QualType DestType) { Info.CCEDiag(E, diag::note_constexpr_overflow) << SrcValue << DestType; return Info.noteUndefinedBehavior(); } static bool HandleFloatToIntCast(EvalInfo &Info, const Expr *E, QualType SrcType, const APFloat &Value, QualType DestType, APSInt &Result) { unsigned DestWidth = Info.Ctx.getIntWidth(DestType); // Determine whether we are converting to unsigned or signed. bool DestSigned = DestType->isSignedIntegerOrEnumerationType(); Result = APSInt(DestWidth, !DestSigned); bool ignored; if (Value.convertToInteger(Result, llvm::APFloat::rmTowardZero, &ignored) & APFloat::opInvalidOp) return HandleOverflow(Info, E, Value, DestType); return true; } static bool HandleFloatToFloatCast(EvalInfo &Info, const Expr *E, QualType SrcType, QualType DestType, APFloat &Result) { APFloat Value = Result; bool ignored; Result.convert(Info.Ctx.getFloatTypeSemantics(DestType), APFloat::rmNearestTiesToEven, &ignored); return true; } static APSInt HandleIntToIntCast(EvalInfo &Info, const Expr *E, QualType DestType, QualType SrcType, const APSInt &Value) { unsigned DestWidth = Info.Ctx.getIntWidth(DestType); // Figure out if this is a truncate, extend or noop cast. // If the input is signed, do a sign extend, noop, or truncate. APSInt Result = Value.extOrTrunc(DestWidth); Result.setIsUnsigned(DestType->isUnsignedIntegerOrEnumerationType()); if (DestType->isBooleanType()) Result = Value.getBoolValue(); return Result; } static bool HandleIntToFloatCast(EvalInfo &Info, const Expr *E, QualType SrcType, const APSInt &Value, QualType DestType, APFloat &Result) { Result = APFloat(Info.Ctx.getFloatTypeSemantics(DestType), 1); Result.convertFromAPInt(Value, Value.isSigned(), APFloat::rmNearestTiesToEven); return true; } static bool truncateBitfieldValue(EvalInfo &Info, const Expr *E, APValue &Value, const FieldDecl *FD) { assert(FD->isBitField() && "truncateBitfieldValue on non-bitfield"); if (!Value.isInt()) { // Trying to store a pointer-cast-to-integer into a bitfield. // FIXME: In this case, we should provide the diagnostic for casting // a pointer to an integer. assert(Value.isLValue() && "integral value neither int nor lvalue?"); Info.FFDiag(E); return false; } APSInt &Int = Value.getInt(); unsigned OldBitWidth = Int.getBitWidth(); unsigned NewBitWidth = FD->getBitWidthValue(Info.Ctx); if (NewBitWidth < OldBitWidth) Int = Int.trunc(NewBitWidth).extend(OldBitWidth); return true; } static bool EvalAndBitcastToAPInt(EvalInfo &Info, const Expr *E, llvm::APInt &Res) { APValue SVal; if (!Evaluate(SVal, Info, E)) return false; if (SVal.isInt()) { Res = SVal.getInt(); return true; } if (SVal.isFloat()) { Res = SVal.getFloat().bitcastToAPInt(); return true; } if (SVal.isVector()) { QualType VecTy = E->getType(); unsigned VecSize = Info.Ctx.getTypeSize(VecTy); QualType EltTy = VecTy->castAs()->getElementType(); unsigned EltSize = Info.Ctx.getTypeSize(EltTy); bool BigEndian = Info.Ctx.getTargetInfo().isBigEndian(); Res = llvm::APInt::getNullValue(VecSize); for (unsigned i = 0; i < SVal.getVectorLength(); i++) { APValue &Elt = SVal.getVectorElt(i); llvm::APInt EltAsInt; if (Elt.isInt()) { EltAsInt = Elt.getInt(); } else if (Elt.isFloat()) { EltAsInt = Elt.getFloat().bitcastToAPInt(); } else { // Don't try to handle vectors of anything other than int or float // (not sure if it's possible to hit this case). Info.FFDiag(E, diag::note_invalid_subexpr_in_const_expr); return false; } unsigned BaseEltSize = EltAsInt.getBitWidth(); if (BigEndian) Res |= EltAsInt.zextOrTrunc(VecSize).rotr(i*EltSize+BaseEltSize); else Res |= EltAsInt.zextOrTrunc(VecSize).rotl(i*EltSize); } return true; } // Give up if the input isn't an int, float, or vector. For example, we // reject "(v4i16)(intptr_t)&a". Info.FFDiag(E, diag::note_invalid_subexpr_in_const_expr); return false; } /// Perform the given integer operation, which is known to need at most BitWidth /// bits, and check for overflow in the original type (if that type was not an /// unsigned type). template static bool CheckedIntArithmetic(EvalInfo &Info, const Expr *E, const APSInt &LHS, const APSInt &RHS, unsigned BitWidth, Operation Op, APSInt &Result) { if (LHS.isUnsigned()) { Result = Op(LHS, RHS); return true; } APSInt Value(Op(LHS.extend(BitWidth), RHS.extend(BitWidth)), false); Result = Value.trunc(LHS.getBitWidth()); if (Result.extend(BitWidth) != Value) { if (Info.checkingForUndefinedBehavior()) Info.Ctx.getDiagnostics().Report(E->getExprLoc(), diag::warn_integer_constant_overflow) << Result.toString(10) << E->getType(); else return HandleOverflow(Info, E, Value, E->getType()); } return true; } /// Perform the given binary integer operation. static bool handleIntIntBinOp(EvalInfo &Info, const Expr *E, const APSInt &LHS, BinaryOperatorKind Opcode, APSInt RHS, APSInt &Result) { switch (Opcode) { default: Info.FFDiag(E); return false; case BO_Mul: return CheckedIntArithmetic(Info, E, LHS, RHS, LHS.getBitWidth() * 2, std::multiplies(), Result); case BO_Add: return CheckedIntArithmetic(Info, E, LHS, RHS, LHS.getBitWidth() + 1, std::plus(), Result); case BO_Sub: return CheckedIntArithmetic(Info, E, LHS, RHS, LHS.getBitWidth() + 1, std::minus(), Result); case BO_And: Result = LHS & RHS; return true; case BO_Xor: Result = LHS ^ RHS; return true; case BO_Or: Result = LHS | RHS; return true; case BO_Div: case BO_Rem: if (RHS == 0) { Info.FFDiag(E, diag::note_expr_divide_by_zero); return false; } Result = (Opcode == BO_Rem ? LHS % RHS : LHS / RHS); // Check for overflow case: INT_MIN / -1 or INT_MIN % -1. APSInt supports // this operation and gives the two's complement result. if (RHS.isNegative() && RHS.isAllOnesValue() && LHS.isSigned() && LHS.isMinSignedValue()) return HandleOverflow(Info, E, -LHS.extend(LHS.getBitWidth() + 1), E->getType()); return true; case BO_Shl: { if (Info.getLangOpts().OpenCL) // OpenCL 6.3j: shift values are effectively % word size of LHS. RHS &= APSInt(llvm::APInt(RHS.getBitWidth(), static_cast(LHS.getBitWidth() - 1)), RHS.isUnsigned()); else if (RHS.isSigned() && RHS.isNegative()) { // During constant-folding, a negative shift is an opposite shift. Such // a shift is not a constant expression. Info.CCEDiag(E, diag::note_constexpr_negative_shift) << RHS; RHS = -RHS; goto shift_right; } shift_left: // C++11 [expr.shift]p1: Shift width must be less than the bit width of // the shifted type. unsigned SA = (unsigned) RHS.getLimitedValue(LHS.getBitWidth()-1); if (SA != RHS) { Info.CCEDiag(E, diag::note_constexpr_large_shift) << RHS << E->getType() << LHS.getBitWidth(); } else if (LHS.isSigned() && !Info.getLangOpts().CPlusPlus2a) { // C++11 [expr.shift]p2: A signed left shift must have a non-negative // operand, and must not overflow the corresponding unsigned type. // C++2a [expr.shift]p2: E1 << E2 is the unique value congruent to // E1 x 2^E2 module 2^N. if (LHS.isNegative()) Info.CCEDiag(E, diag::note_constexpr_lshift_of_negative) << LHS; else if (LHS.countLeadingZeros() < SA) Info.CCEDiag(E, diag::note_constexpr_lshift_discards); } Result = LHS << SA; return true; } case BO_Shr: { if (Info.getLangOpts().OpenCL) // OpenCL 6.3j: shift values are effectively % word size of LHS. RHS &= APSInt(llvm::APInt(RHS.getBitWidth(), static_cast(LHS.getBitWidth() - 1)), RHS.isUnsigned()); else if (RHS.isSigned() && RHS.isNegative()) { // During constant-folding, a negative shift is an opposite shift. Such a // shift is not a constant expression. Info.CCEDiag(E, diag::note_constexpr_negative_shift) << RHS; RHS = -RHS; goto shift_left; } shift_right: // C++11 [expr.shift]p1: Shift width must be less than the bit width of the // shifted type. unsigned SA = (unsigned) RHS.getLimitedValue(LHS.getBitWidth()-1); if (SA != RHS) Info.CCEDiag(E, diag::note_constexpr_large_shift) << RHS << E->getType() << LHS.getBitWidth(); Result = LHS >> SA; return true; } case BO_LT: Result = LHS < RHS; return true; case BO_GT: Result = LHS > RHS; return true; case BO_LE: Result = LHS <= RHS; return true; case BO_GE: Result = LHS >= RHS; return true; case BO_EQ: Result = LHS == RHS; return true; case BO_NE: Result = LHS != RHS; return true; case BO_Cmp: llvm_unreachable("BO_Cmp should be handled elsewhere"); } } /// Perform the given binary floating-point operation, in-place, on LHS. static bool handleFloatFloatBinOp(EvalInfo &Info, const Expr *E, APFloat &LHS, BinaryOperatorKind Opcode, const APFloat &RHS) { switch (Opcode) { default: Info.FFDiag(E); return false; case BO_Mul: LHS.multiply(RHS, APFloat::rmNearestTiesToEven); break; case BO_Add: LHS.add(RHS, APFloat::rmNearestTiesToEven); break; case BO_Sub: LHS.subtract(RHS, APFloat::rmNearestTiesToEven); break; case BO_Div: // [expr.mul]p4: // If the second operand of / or % is zero the behavior is undefined. if (RHS.isZero()) Info.CCEDiag(E, diag::note_expr_divide_by_zero); LHS.divide(RHS, APFloat::rmNearestTiesToEven); break; } // [expr.pre]p4: // If during the evaluation of an expression, the result is not // mathematically defined [...], the behavior is undefined. // FIXME: C++ rules require us to not conform to IEEE 754 here. if (LHS.isNaN()) { Info.CCEDiag(E, diag::note_constexpr_float_arithmetic) << LHS.isNaN(); return Info.noteUndefinedBehavior(); } return true; } /// Cast an lvalue referring to a base subobject to a derived class, by /// truncating the lvalue's path to the given length. static bool CastToDerivedClass(EvalInfo &Info, const Expr *E, LValue &Result, const RecordDecl *TruncatedType, unsigned TruncatedElements) { SubobjectDesignator &D = Result.Designator; // Check we actually point to a derived class object. if (TruncatedElements == D.Entries.size()) return true; assert(TruncatedElements >= D.MostDerivedPathLength && "not casting to a derived class"); if (!Result.checkSubobject(Info, E, CSK_Derived)) return false; // Truncate the path to the subobject, and remove any derived-to-base offsets. const RecordDecl *RD = TruncatedType; for (unsigned I = TruncatedElements, N = D.Entries.size(); I != N; ++I) { if (RD->isInvalidDecl()) return false; const ASTRecordLayout &Layout = Info.Ctx.getASTRecordLayout(RD); const CXXRecordDecl *Base = getAsBaseClass(D.Entries[I]); if (isVirtualBaseClass(D.Entries[I])) Result.Offset -= Layout.getVBaseClassOffset(Base); else Result.Offset -= Layout.getBaseClassOffset(Base); RD = Base; } D.Entries.resize(TruncatedElements); return true; } static bool HandleLValueDirectBase(EvalInfo &Info, const Expr *E, LValue &Obj, const CXXRecordDecl *Derived, const CXXRecordDecl *Base, const ASTRecordLayout *RL = nullptr) { if (!RL) { if (Derived->isInvalidDecl()) return false; RL = &Info.Ctx.getASTRecordLayout(Derived); } Obj.getLValueOffset() += RL->getBaseClassOffset(Base); Obj.addDecl(Info, E, Base, /*Virtual*/ false); return true; } static bool HandleLValueBase(EvalInfo &Info, const Expr *E, LValue &Obj, const CXXRecordDecl *DerivedDecl, const CXXBaseSpecifier *Base) { const CXXRecordDecl *BaseDecl = Base->getType()->getAsCXXRecordDecl(); if (!Base->isVirtual()) return HandleLValueDirectBase(Info, E, Obj, DerivedDecl, BaseDecl); SubobjectDesignator &D = Obj.Designator; if (D.Invalid) return false; // Extract most-derived object and corresponding type. DerivedDecl = D.MostDerivedType->getAsCXXRecordDecl(); if (!CastToDerivedClass(Info, E, Obj, DerivedDecl, D.MostDerivedPathLength)) return false; // Find the virtual base class. if (DerivedDecl->isInvalidDecl()) return false; const ASTRecordLayout &Layout = Info.Ctx.getASTRecordLayout(DerivedDecl); Obj.getLValueOffset() += Layout.getVBaseClassOffset(BaseDecl); Obj.addDecl(Info, E, BaseDecl, /*Virtual*/ true); return true; } static bool HandleLValueBasePath(EvalInfo &Info, const CastExpr *E, QualType Type, LValue &Result) { for (CastExpr::path_const_iterator PathI = E->path_begin(), PathE = E->path_end(); PathI != PathE; ++PathI) { if (!HandleLValueBase(Info, E, Result, Type->getAsCXXRecordDecl(), *PathI)) return false; Type = (*PathI)->getType(); } return true; } /// Cast an lvalue referring to a derived class to a known base subobject. static bool CastToBaseClass(EvalInfo &Info, const Expr *E, LValue &Result, const CXXRecordDecl *DerivedRD, const CXXRecordDecl *BaseRD) { CXXBasePaths Paths(/*FindAmbiguities=*/false, /*RecordPaths=*/true, /*DetectVirtual=*/false); if (!DerivedRD->isDerivedFrom(BaseRD, Paths)) llvm_unreachable("Class must be derived from the passed in base class!"); for (CXXBasePathElement &Elem : Paths.front()) if (!HandleLValueBase(Info, E, Result, Elem.Class, Elem.Base)) return false; return true; } /// Update LVal to refer to the given field, which must be a member of the type /// currently described by LVal. static bool HandleLValueMember(EvalInfo &Info, const Expr *E, LValue &LVal, const FieldDecl *FD, const ASTRecordLayout *RL = nullptr) { if (!RL) { if (FD->getParent()->isInvalidDecl()) return false; RL = &Info.Ctx.getASTRecordLayout(FD->getParent()); } unsigned I = FD->getFieldIndex(); LVal.adjustOffset(Info.Ctx.toCharUnitsFromBits(RL->getFieldOffset(I))); LVal.addDecl(Info, E, FD); return true; } /// Update LVal to refer to the given indirect field. static bool HandleLValueIndirectMember(EvalInfo &Info, const Expr *E, LValue &LVal, const IndirectFieldDecl *IFD) { for (const auto *C : IFD->chain()) if (!HandleLValueMember(Info, E, LVal, cast(C))) return false; return true; } /// Get the size of the given type in char units. static bool HandleSizeof(EvalInfo &Info, SourceLocation Loc, QualType Type, CharUnits &Size) { // sizeof(void), __alignof__(void), sizeof(function) = 1 as a gcc // extension. if (Type->isVoidType() || Type->isFunctionType()) { Size = CharUnits::One(); return true; } if (Type->isDependentType()) { Info.FFDiag(Loc); return false; } if (!Type->isConstantSizeType()) { // sizeof(vla) is not a constantexpr: C99 6.5.3.4p2. // FIXME: Better diagnostic. Info.FFDiag(Loc); return false; } Size = Info.Ctx.getTypeSizeInChars(Type); return true; } /// Update a pointer value to model pointer arithmetic. /// \param Info - Information about the ongoing evaluation. /// \param E - The expression being evaluated, for diagnostic purposes. /// \param LVal - The pointer value to be updated. /// \param EltTy - The pointee type represented by LVal. /// \param Adjustment - The adjustment, in objects of type EltTy, to add. static bool HandleLValueArrayAdjustment(EvalInfo &Info, const Expr *E, LValue &LVal, QualType EltTy, APSInt Adjustment) { CharUnits SizeOfPointee; if (!HandleSizeof(Info, E->getExprLoc(), EltTy, SizeOfPointee)) return false; LVal.adjustOffsetAndIndex(Info, E, Adjustment, SizeOfPointee); return true; } static bool HandleLValueArrayAdjustment(EvalInfo &Info, const Expr *E, LValue &LVal, QualType EltTy, int64_t Adjustment) { return HandleLValueArrayAdjustment(Info, E, LVal, EltTy, APSInt::get(Adjustment)); } /// Update an lvalue to refer to a component of a complex number. /// \param Info - Information about the ongoing evaluation. /// \param LVal - The lvalue to be updated. /// \param EltTy - The complex number's component type. /// \param Imag - False for the real component, true for the imaginary. static bool HandleLValueComplexElement(EvalInfo &Info, const Expr *E, LValue &LVal, QualType EltTy, bool Imag) { if (Imag) { CharUnits SizeOfComponent; if (!HandleSizeof(Info, E->getExprLoc(), EltTy, SizeOfComponent)) return false; LVal.Offset += SizeOfComponent; } LVal.addComplex(Info, E, EltTy, Imag); return true; } /// Try to evaluate the initializer for a variable declaration. /// /// \param Info Information about the ongoing evaluation. /// \param E An expression to be used when printing diagnostics. /// \param VD The variable whose initializer should be obtained. /// \param Frame The frame in which the variable was created. Must be null /// if this variable is not local to the evaluation. /// \param Result Filled in with a pointer to the value of the variable. static bool evaluateVarDeclInit(EvalInfo &Info, const Expr *E, const VarDecl *VD, CallStackFrame *Frame, APValue *&Result, const LValue *LVal) { // If this is a parameter to an active constexpr function call, perform // argument substitution. if (const ParmVarDecl *PVD = dyn_cast(VD)) { // Assume arguments of a potential constant expression are unknown // constant expressions. if (Info.checkingPotentialConstantExpression()) return false; if (!Frame || !Frame->Arguments) { Info.FFDiag(E, diag::note_invalid_subexpr_in_const_expr); return false; } Result = &Frame->Arguments[PVD->getFunctionScopeIndex()]; return true; } // If this is a local variable, dig out its value. if (Frame) { Result = LVal ? Frame->getTemporary(VD, LVal->getLValueVersion()) : Frame->getCurrentTemporary(VD); if (!Result) { // Assume variables referenced within a lambda's call operator that were // not declared within the call operator are captures and during checking // of a potential constant expression, assume they are unknown constant // expressions. assert(isLambdaCallOperator(Frame->Callee) && (VD->getDeclContext() != Frame->Callee || VD->isInitCapture()) && "missing value for local variable"); if (Info.checkingPotentialConstantExpression()) return false; // FIXME: implement capture evaluation during constant expr evaluation. Info.FFDiag(E->getBeginLoc(), diag::note_unimplemented_constexpr_lambda_feature_ast) << "captures not currently allowed"; return false; } return true; } // Dig out the initializer, and use the declaration which it's attached to. const Expr *Init = VD->getAnyInitializer(VD); if (!Init || Init->isValueDependent()) { // If we're checking a potential constant expression, the variable could be // initialized later. if (!Info.checkingPotentialConstantExpression()) Info.FFDiag(E, diag::note_invalid_subexpr_in_const_expr); return false; } // If we're currently evaluating the initializer of this declaration, use that // in-flight value. if (Info.EvaluatingDecl.dyn_cast() == VD) { Result = Info.EvaluatingDeclValue; return true; } // Never evaluate the initializer of a weak variable. We can't be sure that // this is the definition which will be used. if (VD->isWeak()) { Info.FFDiag(E, diag::note_invalid_subexpr_in_const_expr); return false; } // Check that we can fold the initializer. In C++, we will have already done // this in the cases where it matters for conformance. SmallVector Notes; if (!VD->evaluateValue(Notes)) { Info.FFDiag(E, diag::note_constexpr_var_init_non_constant, Notes.size() + 1) << VD; Info.Note(VD->getLocation(), diag::note_declared_at); Info.addNotes(Notes); return false; } else if (!VD->checkInitIsICE()) { Info.CCEDiag(E, diag::note_constexpr_var_init_non_constant, Notes.size() + 1) << VD; Info.Note(VD->getLocation(), diag::note_declared_at); Info.addNotes(Notes); } Result = VD->getEvaluatedValue(); return true; } static bool IsConstNonVolatile(QualType T) { Qualifiers Quals = T.getQualifiers(); return Quals.hasConst() && !Quals.hasVolatile(); } /// Get the base index of the given base class within an APValue representing /// the given derived class. static unsigned getBaseIndex(const CXXRecordDecl *Derived, const CXXRecordDecl *Base) { Base = Base->getCanonicalDecl(); unsigned Index = 0; for (CXXRecordDecl::base_class_const_iterator I = Derived->bases_begin(), E = Derived->bases_end(); I != E; ++I, ++Index) { if (I->getType()->getAsCXXRecordDecl()->getCanonicalDecl() == Base) return Index; } llvm_unreachable("base class missing from derived class's bases list"); } /// Extract the value of a character from a string literal. static APSInt extractStringLiteralCharacter(EvalInfo &Info, const Expr *Lit, uint64_t Index) { assert(!isa(Lit) && "SourceLocExpr should have already been converted to a StringLiteral"); // FIXME: Support MakeStringConstant if (const auto *ObjCEnc = dyn_cast(Lit)) { std::string Str; Info.Ctx.getObjCEncodingForType(ObjCEnc->getEncodedType(), Str); assert(Index <= Str.size() && "Index too large"); return APSInt::getUnsigned(Str.c_str()[Index]); } if (auto PE = dyn_cast(Lit)) Lit = PE->getFunctionName(); const StringLiteral *S = cast(Lit); const ConstantArrayType *CAT = Info.Ctx.getAsConstantArrayType(S->getType()); assert(CAT && "string literal isn't an array"); QualType CharType = CAT->getElementType(); assert(CharType->isIntegerType() && "unexpected character type"); APSInt Value(S->getCharByteWidth() * Info.Ctx.getCharWidth(), CharType->isUnsignedIntegerType()); if (Index < S->getLength()) Value = S->getCodeUnit(Index); return Value; } // Expand a string literal into an array of characters. // // FIXME: This is inefficient; we should probably introduce something similar // to the LLVM ConstantDataArray to make this cheaper. static void expandStringLiteral(EvalInfo &Info, const StringLiteral *S, APValue &Result) { const ConstantArrayType *CAT = Info.Ctx.getAsConstantArrayType(S->getType()); assert(CAT && "string literal isn't an array"); QualType CharType = CAT->getElementType(); assert(CharType->isIntegerType() && "unexpected character type"); unsigned Elts = CAT->getSize().getZExtValue(); Result = APValue(APValue::UninitArray(), std::min(S->getLength(), Elts), Elts); APSInt Value(S->getCharByteWidth() * Info.Ctx.getCharWidth(), CharType->isUnsignedIntegerType()); if (Result.hasArrayFiller()) Result.getArrayFiller() = APValue(Value); for (unsigned I = 0, N = Result.getArrayInitializedElts(); I != N; ++I) { Value = S->getCodeUnit(I); Result.getArrayInitializedElt(I) = APValue(Value); } } // Expand an array so that it has more than Index filled elements. static void expandArray(APValue &Array, unsigned Index) { unsigned Size = Array.getArraySize(); assert(Index < Size); // Always at least double the number of elements for which we store a value. unsigned OldElts = Array.getArrayInitializedElts(); unsigned NewElts = std::max(Index+1, OldElts * 2); NewElts = std::min(Size, std::max(NewElts, 8u)); // Copy the data across. APValue NewValue(APValue::UninitArray(), NewElts, Size); for (unsigned I = 0; I != OldElts; ++I) NewValue.getArrayInitializedElt(I).swap(Array.getArrayInitializedElt(I)); for (unsigned I = OldElts; I != NewElts; ++I) NewValue.getArrayInitializedElt(I) = Array.getArrayFiller(); if (NewValue.hasArrayFiller()) NewValue.getArrayFiller() = Array.getArrayFiller(); Array.swap(NewValue); } /// Determine whether a type would actually be read by an lvalue-to-rvalue /// conversion. If it's of class type, we may assume that the copy operation /// is trivial. Note that this is never true for a union type with fields /// (because the copy always "reads" the active member) and always true for /// a non-class type. static bool isReadByLvalueToRvalueConversion(QualType T) { CXXRecordDecl *RD = T->getBaseElementTypeUnsafe()->getAsCXXRecordDecl(); if (!RD || (RD->isUnion() && !RD->field_empty())) return true; if (RD->isEmpty()) return false; for (auto *Field : RD->fields()) if (isReadByLvalueToRvalueConversion(Field->getType())) return true; for (auto &BaseSpec : RD->bases()) if (isReadByLvalueToRvalueConversion(BaseSpec.getType())) return true; return false; } /// Diagnose an attempt to read from any unreadable field within the specified /// type, which might be a class type. static bool diagnoseUnreadableFields(EvalInfo &Info, const Expr *E, QualType T) { CXXRecordDecl *RD = T->getBaseElementTypeUnsafe()->getAsCXXRecordDecl(); if (!RD) return false; if (!RD->hasMutableFields()) return false; for (auto *Field : RD->fields()) { // If we're actually going to read this field in some way, then it can't // be mutable. If we're in a union, then assigning to a mutable field // (even an empty one) can change the active member, so that's not OK. // FIXME: Add core issue number for the union case. if (Field->isMutable() && (RD->isUnion() || isReadByLvalueToRvalueConversion(Field->getType()))) { Info.FFDiag(E, diag::note_constexpr_ltor_mutable, 1) << Field; Info.Note(Field->getLocation(), diag::note_declared_at); return true; } if (diagnoseUnreadableFields(Info, E, Field->getType())) return true; } for (auto &BaseSpec : RD->bases()) if (diagnoseUnreadableFields(Info, E, BaseSpec.getType())) return true; // All mutable fields were empty, and thus not actually read. return false; } static bool lifetimeStartedInEvaluation(EvalInfo &Info, APValue::LValueBase Base) { // A temporary we created. if (Base.getCallIndex()) return true; auto *Evaluating = Info.EvaluatingDecl.dyn_cast(); if (!Evaluating) return false; // The variable whose initializer we're evaluating. if (auto *BaseD = Base.dyn_cast()) if (declaresSameEntity(Evaluating, BaseD)) return true; // A temporary lifetime-extended by the variable whose initializer we're // evaluating. if (auto *BaseE = Base.dyn_cast()) if (auto *BaseMTE = dyn_cast(BaseE)) if (declaresSameEntity(BaseMTE->getExtendingDecl(), Evaluating)) return true; return false; } namespace { /// A handle to a complete object (an object that is not a subobject of /// another object). struct CompleteObject { /// The identity of the object. APValue::LValueBase Base; /// The value of the complete object. APValue *Value; /// The type of the complete object. QualType Type; CompleteObject() : Value(nullptr) {} CompleteObject(APValue::LValueBase Base, APValue *Value, QualType Type) : Base(Base), Value(Value), Type(Type) {} bool mayReadMutableMembers(EvalInfo &Info) const { // In C++14 onwards, it is permitted to read a mutable member whose // lifetime began within the evaluation. // FIXME: Should we also allow this in C++11? if (!Info.getLangOpts().CPlusPlus14) return false; return lifetimeStartedInEvaluation(Info, Base); } explicit operator bool() const { return !Type.isNull(); } }; } // end anonymous namespace static QualType getSubobjectType(QualType ObjType, QualType SubobjType, bool IsMutable = false) { // C++ [basic.type.qualifier]p1: // - A const object is an object of type const T or a non-mutable subobject // of a const object. if (ObjType.isConstQualified() && !IsMutable) SubobjType.addConst(); // - A volatile object is an object of type const T or a subobject of a // volatile object. if (ObjType.isVolatileQualified()) SubobjType.addVolatile(); return SubobjType; } /// Find the designated sub-object of an rvalue. template typename SubobjectHandler::result_type findSubobject(EvalInfo &Info, const Expr *E, const CompleteObject &Obj, const SubobjectDesignator &Sub, SubobjectHandler &handler) { if (Sub.Invalid) // A diagnostic will have already been produced. return handler.failed(); if (Sub.isOnePastTheEnd() || Sub.isMostDerivedAnUnsizedArray()) { if (Info.getLangOpts().CPlusPlus11) Info.FFDiag(E, Sub.isOnePastTheEnd() ? diag::note_constexpr_access_past_end : diag::note_constexpr_access_unsized_array) << handler.AccessKind; else Info.FFDiag(E); return handler.failed(); } APValue *O = Obj.Value; QualType ObjType = Obj.Type; const FieldDecl *LastField = nullptr; const FieldDecl *VolatileField = nullptr; // Walk the designator's path to find the subobject. for (unsigned I = 0, N = Sub.Entries.size(); /**/; ++I) { // Reading an indeterminate value is undefined, but assigning over one is OK. if (O->isAbsent() || (O->isIndeterminate() && handler.AccessKind != AK_Assign)) { if (!Info.checkingPotentialConstantExpression()) Info.FFDiag(E, diag::note_constexpr_access_uninit) << handler.AccessKind << O->isIndeterminate(); return handler.failed(); } // C++ [class.ctor]p5: // const and volatile semantics are not applied on an object under // construction. if ((ObjType.isConstQualified() || ObjType.isVolatileQualified()) && ObjType->isRecordType() && Info.isEvaluatingConstructor( Obj.Base, llvm::makeArrayRef(Sub.Entries.begin(), Sub.Entries.begin() + I)) != ConstructionPhase::None) { ObjType = Info.Ctx.getCanonicalType(ObjType); ObjType.removeLocalConst(); ObjType.removeLocalVolatile(); } // If this is our last pass, check that the final object type is OK. if (I == N || (I == N - 1 && ObjType->isAnyComplexType())) { // Accesses to volatile objects are prohibited. if (ObjType.isVolatileQualified() && isFormalAccess(handler.AccessKind)) { if (Info.getLangOpts().CPlusPlus) { int DiagKind; SourceLocation Loc; const NamedDecl *Decl = nullptr; if (VolatileField) { DiagKind = 2; Loc = VolatileField->getLocation(); Decl = VolatileField; } else if (auto *VD = Obj.Base.dyn_cast()) { DiagKind = 1; Loc = VD->getLocation(); Decl = VD; } else { DiagKind = 0; if (auto *E = Obj.Base.dyn_cast()) Loc = E->getExprLoc(); } Info.FFDiag(E, diag::note_constexpr_access_volatile_obj, 1) << handler.AccessKind << DiagKind << Decl; Info.Note(Loc, diag::note_constexpr_volatile_here) << DiagKind; } else { Info.FFDiag(E, diag::note_invalid_subexpr_in_const_expr); } return handler.failed(); } // If we are reading an object of class type, there may still be more // things we need to check: if there are any mutable subobjects, we // cannot perform this read. (This only happens when performing a trivial // copy or assignment.) if (ObjType->isRecordType() && handler.AccessKind == AK_Read && !Obj.mayReadMutableMembers(Info) && diagnoseUnreadableFields(Info, E, ObjType)) return handler.failed(); } if (I == N) { if (!handler.found(*O, ObjType)) return false; // If we modified a bit-field, truncate it to the right width. if (isModification(handler.AccessKind) && LastField && LastField->isBitField() && !truncateBitfieldValue(Info, E, *O, LastField)) return false; return true; } LastField = nullptr; if (ObjType->isArrayType()) { // Next subobject is an array element. const ConstantArrayType *CAT = Info.Ctx.getAsConstantArrayType(ObjType); assert(CAT && "vla in literal type?"); uint64_t Index = Sub.Entries[I].getAsArrayIndex(); if (CAT->getSize().ule(Index)) { // Note, it should not be possible to form a pointer with a valid // designator which points more than one past the end of the array. if (Info.getLangOpts().CPlusPlus11) Info.FFDiag(E, diag::note_constexpr_access_past_end) << handler.AccessKind; else Info.FFDiag(E); return handler.failed(); } ObjType = CAT->getElementType(); if (O->getArrayInitializedElts() > Index) O = &O->getArrayInitializedElt(Index); else if (handler.AccessKind != AK_Read) { expandArray(*O, Index); O = &O->getArrayInitializedElt(Index); } else O = &O->getArrayFiller(); } else if (ObjType->isAnyComplexType()) { // Next subobject is a complex number. uint64_t Index = Sub.Entries[I].getAsArrayIndex(); if (Index > 1) { if (Info.getLangOpts().CPlusPlus11) Info.FFDiag(E, diag::note_constexpr_access_past_end) << handler.AccessKind; else Info.FFDiag(E); return handler.failed(); } ObjType = getSubobjectType( ObjType, ObjType->castAs()->getElementType()); assert(I == N - 1 && "extracting subobject of scalar?"); if (O->isComplexInt()) { return handler.found(Index ? O->getComplexIntImag() : O->getComplexIntReal(), ObjType); } else { assert(O->isComplexFloat()); return handler.found(Index ? O->getComplexFloatImag() : O->getComplexFloatReal(), ObjType); } } else if (const FieldDecl *Field = getAsField(Sub.Entries[I])) { if (Field->isMutable() && handler.AccessKind == AK_Read && !Obj.mayReadMutableMembers(Info)) { Info.FFDiag(E, diag::note_constexpr_ltor_mutable, 1) << Field; Info.Note(Field->getLocation(), diag::note_declared_at); return handler.failed(); } // Next subobject is a class, struct or union field. RecordDecl *RD = ObjType->castAs()->getDecl(); if (RD->isUnion()) { const FieldDecl *UnionField = O->getUnionField(); if (!UnionField || UnionField->getCanonicalDecl() != Field->getCanonicalDecl()) { Info.FFDiag(E, diag::note_constexpr_access_inactive_union_member) << handler.AccessKind << Field << !UnionField << UnionField; return handler.failed(); } O = &O->getUnionValue(); } else O = &O->getStructField(Field->getFieldIndex()); ObjType = getSubobjectType(ObjType, Field->getType(), Field->isMutable()); LastField = Field; if (Field->getType().isVolatileQualified()) VolatileField = Field; } else { // Next subobject is a base class. const CXXRecordDecl *Derived = ObjType->getAsCXXRecordDecl(); const CXXRecordDecl *Base = getAsBaseClass(Sub.Entries[I]); O = &O->getStructBase(getBaseIndex(Derived, Base)); ObjType = getSubobjectType(ObjType, Info.Ctx.getRecordType(Base)); } } } namespace { struct ExtractSubobjectHandler { EvalInfo &Info; APValue &Result; static const AccessKinds AccessKind = AK_Read; typedef bool result_type; bool failed() { return false; } bool found(APValue &Subobj, QualType SubobjType) { Result = Subobj; return true; } bool found(APSInt &Value, QualType SubobjType) { Result = APValue(Value); return true; } bool found(APFloat &Value, QualType SubobjType) { Result = APValue(Value); return true; } }; } // end anonymous namespace const AccessKinds ExtractSubobjectHandler::AccessKind; /// Extract the designated sub-object of an rvalue. static bool extractSubobject(EvalInfo &Info, const Expr *E, const CompleteObject &Obj, const SubobjectDesignator &Sub, APValue &Result) { ExtractSubobjectHandler Handler = { Info, Result }; return findSubobject(Info, E, Obj, Sub, Handler); } namespace { struct ModifySubobjectHandler { EvalInfo &Info; APValue &NewVal; const Expr *E; typedef bool result_type; static const AccessKinds AccessKind = AK_Assign; bool checkConst(QualType QT) { // Assigning to a const object has undefined behavior. if (QT.isConstQualified()) { Info.FFDiag(E, diag::note_constexpr_modify_const_type) << QT; return false; } return true; } bool failed() { return false; } bool found(APValue &Subobj, QualType SubobjType) { if (!checkConst(SubobjType)) return false; // We've been given ownership of NewVal, so just swap it in. Subobj.swap(NewVal); return true; } bool found(APSInt &Value, QualType SubobjType) { if (!checkConst(SubobjType)) return false; if (!NewVal.isInt()) { // Maybe trying to write a cast pointer value into a complex? Info.FFDiag(E); return false; } Value = NewVal.getInt(); return true; } bool found(APFloat &Value, QualType SubobjType) { if (!checkConst(SubobjType)) return false; Value = NewVal.getFloat(); return true; } }; } // end anonymous namespace const AccessKinds ModifySubobjectHandler::AccessKind; /// Update the designated sub-object of an rvalue to the given value. static bool modifySubobject(EvalInfo &Info, const Expr *E, const CompleteObject &Obj, const SubobjectDesignator &Sub, APValue &NewVal) { ModifySubobjectHandler Handler = { Info, NewVal, E }; return findSubobject(Info, E, Obj, Sub, Handler); } /// Find the position where two subobject designators diverge, or equivalently /// the length of the common initial subsequence. static unsigned FindDesignatorMismatch(QualType ObjType, const SubobjectDesignator &A, const SubobjectDesignator &B, bool &WasArrayIndex) { unsigned I = 0, N = std::min(A.Entries.size(), B.Entries.size()); for (/**/; I != N; ++I) { if (!ObjType.isNull() && (ObjType->isArrayType() || ObjType->isAnyComplexType())) { // Next subobject is an array element. if (A.Entries[I].getAsArrayIndex() != B.Entries[I].getAsArrayIndex()) { WasArrayIndex = true; return I; } if (ObjType->isAnyComplexType()) ObjType = ObjType->castAs()->getElementType(); else ObjType = ObjType->castAsArrayTypeUnsafe()->getElementType(); } else { if (A.Entries[I].getAsBaseOrMember() != B.Entries[I].getAsBaseOrMember()) { WasArrayIndex = false; return I; } if (const FieldDecl *FD = getAsField(A.Entries[I])) // Next subobject is a field. ObjType = FD->getType(); else // Next subobject is a base class. ObjType = QualType(); } } WasArrayIndex = false; return I; } /// Determine whether the given subobject designators refer to elements of the /// same array object. static bool AreElementsOfSameArray(QualType ObjType, const SubobjectDesignator &A, const SubobjectDesignator &B) { if (A.Entries.size() != B.Entries.size()) return false; bool IsArray = A.MostDerivedIsArrayElement; if (IsArray && A.MostDerivedPathLength != A.Entries.size()) // A is a subobject of the array element. return false; // If A (and B) designates an array element, the last entry will be the array // index. That doesn't have to match. Otherwise, we're in the 'implicit array // of length 1' case, and the entire path must match. bool WasArrayIndex; unsigned CommonLength = FindDesignatorMismatch(ObjType, A, B, WasArrayIndex); return CommonLength >= A.Entries.size() - IsArray; } /// Find the complete object to which an LValue refers. static CompleteObject findCompleteObject(EvalInfo &Info, const Expr *E, AccessKinds AK, const LValue &LVal, QualType LValType) { if (LVal.InvalidBase) { Info.FFDiag(E); return CompleteObject(); } if (!LVal.Base) { Info.FFDiag(E, diag::note_constexpr_access_null) << AK; return CompleteObject(); } CallStackFrame *Frame = nullptr; unsigned Depth = 0; if (LVal.getLValueCallIndex()) { std::tie(Frame, Depth) = Info.getCallFrameAndDepth(LVal.getLValueCallIndex()); if (!Frame) { Info.FFDiag(E, diag::note_constexpr_lifetime_ended, 1) << AK << LVal.Base.is(); NoteLValueLocation(Info, LVal.Base); return CompleteObject(); } } bool IsAccess = isFormalAccess(AK); // C++11 DR1311: An lvalue-to-rvalue conversion on a volatile-qualified type // is not a constant expression (even if the object is non-volatile). We also // apply this rule to C++98, in order to conform to the expected 'volatile' // semantics. if (IsAccess && LValType.isVolatileQualified()) { if (Info.getLangOpts().CPlusPlus) Info.FFDiag(E, diag::note_constexpr_access_volatile_type) << AK << LValType; else Info.FFDiag(E); return CompleteObject(); } // Compute value storage location and type of base object. APValue *BaseVal = nullptr; QualType BaseType = getType(LVal.Base); if (const ValueDecl *D = LVal.Base.dyn_cast()) { // In C++98, const, non-volatile integers initialized with ICEs are ICEs. // In C++11, constexpr, non-volatile variables initialized with constant // expressions are constant expressions too. Inside constexpr functions, // parameters are constant expressions even if they're non-const. // In C++1y, objects local to a constant expression (those with a Frame) are // both readable and writable inside constant expressions. // In C, such things can also be folded, although they are not ICEs. const VarDecl *VD = dyn_cast(D); if (VD) { if (const VarDecl *VDef = VD->getDefinition(Info.Ctx)) VD = VDef; } if (!VD || VD->isInvalidDecl()) { Info.FFDiag(E); return CompleteObject(); } // Unless we're looking at a local variable or argument in a constexpr call, // the variable we're reading must be const. if (!Frame) { if (Info.getLangOpts().CPlusPlus14 && declaresSameEntity( VD, Info.EvaluatingDecl.dyn_cast())) { // OK, we can read and modify an object if we're in the process of // evaluating its initializer, because its lifetime began in this // evaluation. } else if (isModification(AK)) { // All the remaining cases do not permit modification of the object. Info.FFDiag(E, diag::note_constexpr_modify_global); return CompleteObject(); } else if (VD->isConstexpr()) { // OK, we can read this variable. } else if (BaseType->isIntegralOrEnumerationType()) { // In OpenCL if a variable is in constant address space it is a const // value. if (!(BaseType.isConstQualified() || (Info.getLangOpts().OpenCL && BaseType.getAddressSpace() == LangAS::opencl_constant))) { if (!IsAccess) return CompleteObject(LVal.getLValueBase(), nullptr, BaseType); if (Info.getLangOpts().CPlusPlus) { Info.FFDiag(E, diag::note_constexpr_ltor_non_const_int, 1) << VD; Info.Note(VD->getLocation(), diag::note_declared_at); } else { Info.FFDiag(E); } return CompleteObject(); } } else if (!IsAccess) { return CompleteObject(LVal.getLValueBase(), nullptr, BaseType); } else if (BaseType->isFloatingType() && BaseType.isConstQualified()) { // We support folding of const floating-point types, in order to make // static const data members of such types (supported as an extension) // more useful. if (Info.getLangOpts().CPlusPlus11) { Info.CCEDiag(E, diag::note_constexpr_ltor_non_constexpr, 1) << VD; Info.Note(VD->getLocation(), diag::note_declared_at); } else { Info.CCEDiag(E); } } else if (BaseType.isConstQualified() && VD->hasDefinition(Info.Ctx)) { Info.CCEDiag(E, diag::note_constexpr_ltor_non_constexpr) << VD; // Keep evaluating to see what we can do. } else { // FIXME: Allow folding of values of any literal type in all languages. if (Info.checkingPotentialConstantExpression() && VD->getType().isConstQualified() && !VD->hasDefinition(Info.Ctx)) { // The definition of this variable could be constexpr. We can't // access it right now, but may be able to in future. } else if (Info.getLangOpts().CPlusPlus11) { Info.FFDiag(E, diag::note_constexpr_ltor_non_constexpr, 1) << VD; Info.Note(VD->getLocation(), diag::note_declared_at); } else { Info.FFDiag(E); } return CompleteObject(); } } if (!evaluateVarDeclInit(Info, E, VD, Frame, BaseVal, &LVal)) return CompleteObject(); } else { const Expr *Base = LVal.Base.dyn_cast(); if (!Frame) { if (const MaterializeTemporaryExpr *MTE = dyn_cast_or_null(Base)) { assert(MTE->getStorageDuration() == SD_Static && "should have a frame for a non-global materialized temporary"); // Per C++1y [expr.const]p2: // an lvalue-to-rvalue conversion [is not allowed unless it applies to] // - a [...] glvalue of integral or enumeration type that refers to // a non-volatile const object [...] // [...] // - a [...] glvalue of literal type that refers to a non-volatile // object whose lifetime began within the evaluation of e. // // C++11 misses the 'began within the evaluation of e' check and // instead allows all temporaries, including things like: // int &&r = 1; // int x = ++r; // constexpr int k = r; // Therefore we use the C++14 rules in C++11 too. const ValueDecl *VD = Info.EvaluatingDecl.dyn_cast(); const ValueDecl *ED = MTE->getExtendingDecl(); if (!(BaseType.isConstQualified() && BaseType->isIntegralOrEnumerationType()) && !(VD && VD->getCanonicalDecl() == ED->getCanonicalDecl())) { if (!IsAccess) return CompleteObject(LVal.getLValueBase(), nullptr, BaseType); Info.FFDiag(E, diag::note_constexpr_access_static_temporary, 1) << AK; Info.Note(MTE->getExprLoc(), diag::note_constexpr_temporary_here); return CompleteObject(); } BaseVal = Info.Ctx.getMaterializedTemporaryValue(MTE, false); assert(BaseVal && "got reference to unevaluated temporary"); } else { if (!IsAccess) return CompleteObject(LVal.getLValueBase(), nullptr, BaseType); APValue Val; LVal.moveInto(Val); Info.FFDiag(E, diag::note_constexpr_access_unreadable_object) << AK << Val.getAsString(Info.Ctx, Info.Ctx.getLValueReferenceType(LValType)); NoteLValueLocation(Info, LVal.Base); return CompleteObject(); } } else { BaseVal = Frame->getTemporary(Base, LVal.Base.getVersion()); assert(BaseVal && "missing value for temporary"); } } // In C++14, we can't safely access any mutable state when we might be // evaluating after an unmodeled side effect. // // FIXME: Not all local state is mutable. Allow local constant subobjects // to be read here (but take care with 'mutable' fields). if ((Frame && Info.getLangOpts().CPlusPlus14 && Info.EvalStatus.HasSideEffects) || (isModification(AK) && Depth < Info.SpeculativeEvaluationDepth)) return CompleteObject(); return CompleteObject(LVal.getLValueBase(), BaseVal, BaseType); } /// Perform an lvalue-to-rvalue conversion on the given glvalue. This /// can also be used for 'lvalue-to-lvalue' conversions for looking up the /// glvalue referred to by an entity of reference type. /// /// \param Info - Information about the ongoing evaluation. /// \param Conv - The expression for which we are performing the conversion. /// Used for diagnostics. /// \param Type - The type of the glvalue (before stripping cv-qualifiers in the /// case of a non-class type). /// \param LVal - The glvalue on which we are attempting to perform this action. /// \param RVal - The produced value will be placed here. static bool handleLValueToRValueConversion(EvalInfo &Info, const Expr *Conv, QualType Type, const LValue &LVal, APValue &RVal) { if (LVal.Designator.Invalid) return false; // Check for special cases where there is no existing APValue to look at. const Expr *Base = LVal.Base.dyn_cast(); if (Base && !LVal.getLValueCallIndex() && !Type.isVolatileQualified()) { if (const CompoundLiteralExpr *CLE = dyn_cast(Base)) { // In C99, a CompoundLiteralExpr is an lvalue, and we defer evaluating the // initializer until now for such expressions. Such an expression can't be // an ICE in C, so this only matters for fold. if (Type.isVolatileQualified()) { Info.FFDiag(Conv); return false; } APValue Lit; if (!Evaluate(Lit, Info, CLE->getInitializer())) return false; CompleteObject LitObj(LVal.Base, &Lit, Base->getType()); return extractSubobject(Info, Conv, LitObj, LVal.Designator, RVal); } else if (isa(Base) || isa(Base)) { // Special-case character extraction so we don't have to construct an // APValue for the whole string. assert(LVal.Designator.Entries.size() <= 1 && "Can only read characters from string literals"); if (LVal.Designator.Entries.empty()) { // Fail for now for LValue to RValue conversion of an array. // (This shouldn't show up in C/C++, but it could be triggered by a // weird EvaluateAsRValue call from a tool.) Info.FFDiag(Conv); return false; } if (LVal.Designator.isOnePastTheEnd()) { if (Info.getLangOpts().CPlusPlus11) Info.FFDiag(Conv, diag::note_constexpr_access_past_end) << AK_Read; else Info.FFDiag(Conv); return false; } uint64_t CharIndex = LVal.Designator.Entries[0].getAsArrayIndex(); RVal = APValue(extractStringLiteralCharacter(Info, Base, CharIndex)); return true; } } CompleteObject Obj = findCompleteObject(Info, Conv, AK_Read, LVal, Type); return Obj && extractSubobject(Info, Conv, Obj, LVal.Designator, RVal); } /// Perform an assignment of Val to LVal. Takes ownership of Val. static bool handleAssignment(EvalInfo &Info, const Expr *E, const LValue &LVal, QualType LValType, APValue &Val) { if (LVal.Designator.Invalid) return false; if (!Info.getLangOpts().CPlusPlus14) { Info.FFDiag(E); return false; } CompleteObject Obj = findCompleteObject(Info, E, AK_Assign, LVal, LValType); return Obj && modifySubobject(Info, E, Obj, LVal.Designator, Val); } namespace { struct CompoundAssignSubobjectHandler { EvalInfo &Info; const Expr *E; QualType PromotedLHSType; BinaryOperatorKind Opcode; const APValue &RHS; static const AccessKinds AccessKind = AK_Assign; typedef bool result_type; bool checkConst(QualType QT) { // Assigning to a const object has undefined behavior. if (QT.isConstQualified()) { Info.FFDiag(E, diag::note_constexpr_modify_const_type) << QT; return false; } return true; } bool failed() { return false; } bool found(APValue &Subobj, QualType SubobjType) { switch (Subobj.getKind()) { case APValue::Int: return found(Subobj.getInt(), SubobjType); case APValue::Float: return found(Subobj.getFloat(), SubobjType); case APValue::ComplexInt: case APValue::ComplexFloat: // FIXME: Implement complex compound assignment. Info.FFDiag(E); return false; case APValue::LValue: return foundPointer(Subobj, SubobjType); default: // FIXME: can this happen? Info.FFDiag(E); return false; } } bool found(APSInt &Value, QualType SubobjType) { if (!checkConst(SubobjType)) return false; if (!SubobjType->isIntegerType()) { // We don't support compound assignment on integer-cast-to-pointer // values. Info.FFDiag(E); return false; } if (RHS.isInt()) { APSInt LHS = HandleIntToIntCast(Info, E, PromotedLHSType, SubobjType, Value); if (!handleIntIntBinOp(Info, E, LHS, Opcode, RHS.getInt(), LHS)) return false; Value = HandleIntToIntCast(Info, E, SubobjType, PromotedLHSType, LHS); return true; } else if (RHS.isFloat()) { APFloat FValue(0.0); return HandleIntToFloatCast(Info, E, SubobjType, Value, PromotedLHSType, FValue) && handleFloatFloatBinOp(Info, E, FValue, Opcode, RHS.getFloat()) && HandleFloatToIntCast(Info, E, PromotedLHSType, FValue, SubobjType, Value); } Info.FFDiag(E); return false; } bool found(APFloat &Value, QualType SubobjType) { return checkConst(SubobjType) && HandleFloatToFloatCast(Info, E, SubobjType, PromotedLHSType, Value) && handleFloatFloatBinOp(Info, E, Value, Opcode, RHS.getFloat()) && HandleFloatToFloatCast(Info, E, PromotedLHSType, SubobjType, Value); } bool foundPointer(APValue &Subobj, QualType SubobjType) { if (!checkConst(SubobjType)) return false; QualType PointeeType; if (const PointerType *PT = SubobjType->getAs()) PointeeType = PT->getPointeeType(); if (PointeeType.isNull() || !RHS.isInt() || (Opcode != BO_Add && Opcode != BO_Sub)) { Info.FFDiag(E); return false; } APSInt Offset = RHS.getInt(); if (Opcode == BO_Sub) negateAsSigned(Offset); LValue LVal; LVal.setFrom(Info.Ctx, Subobj); if (!HandleLValueArrayAdjustment(Info, E, LVal, PointeeType, Offset)) return false; LVal.moveInto(Subobj); return true; } }; } // end anonymous namespace const AccessKinds CompoundAssignSubobjectHandler::AccessKind; /// Perform a compound assignment of LVal = RVal. static bool handleCompoundAssignment( EvalInfo &Info, const Expr *E, const LValue &LVal, QualType LValType, QualType PromotedLValType, BinaryOperatorKind Opcode, const APValue &RVal) { if (LVal.Designator.Invalid) return false; if (!Info.getLangOpts().CPlusPlus14) { Info.FFDiag(E); return false; } CompleteObject Obj = findCompleteObject(Info, E, AK_Assign, LVal, LValType); CompoundAssignSubobjectHandler Handler = { Info, E, PromotedLValType, Opcode, RVal }; return Obj && findSubobject(Info, E, Obj, LVal.Designator, Handler); } namespace { struct IncDecSubobjectHandler { EvalInfo &Info; const UnaryOperator *E; AccessKinds AccessKind; APValue *Old; typedef bool result_type; bool checkConst(QualType QT) { // Assigning to a const object has undefined behavior. if (QT.isConstQualified()) { Info.FFDiag(E, diag::note_constexpr_modify_const_type) << QT; return false; } return true; } bool failed() { return false; } bool found(APValue &Subobj, QualType SubobjType) { // Stash the old value. Also clear Old, so we don't clobber it later // if we're post-incrementing a complex. if (Old) { *Old = Subobj; Old = nullptr; } switch (Subobj.getKind()) { case APValue::Int: return found(Subobj.getInt(), SubobjType); case APValue::Float: return found(Subobj.getFloat(), SubobjType); case APValue::ComplexInt: return found(Subobj.getComplexIntReal(), SubobjType->castAs()->getElementType() .withCVRQualifiers(SubobjType.getCVRQualifiers())); case APValue::ComplexFloat: return found(Subobj.getComplexFloatReal(), SubobjType->castAs()->getElementType() .withCVRQualifiers(SubobjType.getCVRQualifiers())); case APValue::LValue: return foundPointer(Subobj, SubobjType); default: // FIXME: can this happen? Info.FFDiag(E); return false; } } bool found(APSInt &Value, QualType SubobjType) { if (!checkConst(SubobjType)) return false; if (!SubobjType->isIntegerType()) { // We don't support increment / decrement on integer-cast-to-pointer // values. Info.FFDiag(E); return false; } if (Old) *Old = APValue(Value); // bool arithmetic promotes to int, and the conversion back to bool // doesn't reduce mod 2^n, so special-case it. if (SubobjType->isBooleanType()) { if (AccessKind == AK_Increment) Value = 1; else Value = !Value; return true; } bool WasNegative = Value.isNegative(); if (AccessKind == AK_Increment) { ++Value; if (!WasNegative && Value.isNegative() && E->canOverflow()) { APSInt ActualValue(Value, /*IsUnsigned*/true); return HandleOverflow(Info, E, ActualValue, SubobjType); } } else { --Value; if (WasNegative && !Value.isNegative() && E->canOverflow()) { unsigned BitWidth = Value.getBitWidth(); APSInt ActualValue(Value.sext(BitWidth + 1), /*IsUnsigned*/false); ActualValue.setBit(BitWidth); return HandleOverflow(Info, E, ActualValue, SubobjType); } } return true; } bool found(APFloat &Value, QualType SubobjType) { if (!checkConst(SubobjType)) return false; if (Old) *Old = APValue(Value); APFloat One(Value.getSemantics(), 1); if (AccessKind == AK_Increment) Value.add(One, APFloat::rmNearestTiesToEven); else Value.subtract(One, APFloat::rmNearestTiesToEven); return true; } bool foundPointer(APValue &Subobj, QualType SubobjType) { if (!checkConst(SubobjType)) return false; QualType PointeeType; if (const PointerType *PT = SubobjType->getAs()) PointeeType = PT->getPointeeType(); else { Info.FFDiag(E); return false; } LValue LVal; LVal.setFrom(Info.Ctx, Subobj); if (!HandleLValueArrayAdjustment(Info, E, LVal, PointeeType, AccessKind == AK_Increment ? 1 : -1)) return false; LVal.moveInto(Subobj); return true; } }; } // end anonymous namespace /// Perform an increment or decrement on LVal. static bool handleIncDec(EvalInfo &Info, const Expr *E, const LValue &LVal, QualType LValType, bool IsIncrement, APValue *Old) { if (LVal.Designator.Invalid) return false; if (!Info.getLangOpts().CPlusPlus14) { Info.FFDiag(E); return false; } AccessKinds AK = IsIncrement ? AK_Increment : AK_Decrement; CompleteObject Obj = findCompleteObject(Info, E, AK, LVal, LValType); IncDecSubobjectHandler Handler = {Info, cast(E), AK, Old}; return Obj && findSubobject(Info, E, Obj, LVal.Designator, Handler); } /// Build an lvalue for the object argument of a member function call. static bool EvaluateObjectArgument(EvalInfo &Info, const Expr *Object, LValue &This) { if (Object->getType()->isPointerType()) return EvaluatePointer(Object, This, Info); if (Object->isGLValue()) return EvaluateLValue(Object, This, Info); if (Object->getType()->isLiteralType(Info.Ctx)) return EvaluateTemporary(Object, This, Info); Info.FFDiag(Object, diag::note_constexpr_nonliteral) << Object->getType(); return false; } /// HandleMemberPointerAccess - Evaluate a member access operation and build an /// lvalue referring to the result. /// /// \param Info - Information about the ongoing evaluation. /// \param LV - An lvalue referring to the base of the member pointer. /// \param RHS - The member pointer expression. /// \param IncludeMember - Specifies whether the member itself is included in /// the resulting LValue subobject designator. This is not possible when /// creating a bound member function. /// \return The field or method declaration to which the member pointer refers, /// or 0 if evaluation fails. static const ValueDecl *HandleMemberPointerAccess(EvalInfo &Info, QualType LVType, LValue &LV, const Expr *RHS, bool IncludeMember = true) { MemberPtr MemPtr; if (!EvaluateMemberPointer(RHS, MemPtr, Info)) return nullptr; // C++11 [expr.mptr.oper]p6: If the second operand is the null pointer to // member value, the behavior is undefined. if (!MemPtr.getDecl()) { // FIXME: Specific diagnostic. Info.FFDiag(RHS); return nullptr; } if (MemPtr.isDerivedMember()) { // This is a member of some derived class. Truncate LV appropriately. // The end of the derived-to-base path for the base object must match the // derived-to-base path for the member pointer. if (LV.Designator.MostDerivedPathLength + MemPtr.Path.size() > LV.Designator.Entries.size()) { Info.FFDiag(RHS); return nullptr; } unsigned PathLengthToMember = LV.Designator.Entries.size() - MemPtr.Path.size(); for (unsigned I = 0, N = MemPtr.Path.size(); I != N; ++I) { const CXXRecordDecl *LVDecl = getAsBaseClass( LV.Designator.Entries[PathLengthToMember + I]); const CXXRecordDecl *MPDecl = MemPtr.Path[I]; if (LVDecl->getCanonicalDecl() != MPDecl->getCanonicalDecl()) { Info.FFDiag(RHS); return nullptr; } } // Truncate the lvalue to the appropriate derived class. if (!CastToDerivedClass(Info, RHS, LV, MemPtr.getContainingRecord(), PathLengthToMember)) return nullptr; } else if (!MemPtr.Path.empty()) { // Extend the LValue path with the member pointer's path. LV.Designator.Entries.reserve(LV.Designator.Entries.size() + MemPtr.Path.size() + IncludeMember); // Walk down to the appropriate base class. if (const PointerType *PT = LVType->getAs()) LVType = PT->getPointeeType(); const CXXRecordDecl *RD = LVType->getAsCXXRecordDecl(); assert(RD && "member pointer access on non-class-type expression"); // The first class in the path is that of the lvalue. for (unsigned I = 1, N = MemPtr.Path.size(); I != N; ++I) { const CXXRecordDecl *Base = MemPtr.Path[N - I - 1]; if (!HandleLValueDirectBase(Info, RHS, LV, RD, Base)) return nullptr; RD = Base; } // Finally cast to the class containing the member. if (!HandleLValueDirectBase(Info, RHS, LV, RD, MemPtr.getContainingRecord())) return nullptr; } // Add the member. Note that we cannot build bound member functions here. if (IncludeMember) { if (const FieldDecl *FD = dyn_cast(MemPtr.getDecl())) { if (!HandleLValueMember(Info, RHS, LV, FD)) return nullptr; } else if (const IndirectFieldDecl *IFD = dyn_cast(MemPtr.getDecl())) { if (!HandleLValueIndirectMember(Info, RHS, LV, IFD)) return nullptr; } else { llvm_unreachable("can't construct reference to bound member function"); } } return MemPtr.getDecl(); } static const ValueDecl *HandleMemberPointerAccess(EvalInfo &Info, const BinaryOperator *BO, LValue &LV, bool IncludeMember = true) { assert(BO->getOpcode() == BO_PtrMemD || BO->getOpcode() == BO_PtrMemI); if (!EvaluateObjectArgument(Info, BO->getLHS(), LV)) { if (Info.noteFailure()) { MemberPtr MemPtr; EvaluateMemberPointer(BO->getRHS(), MemPtr, Info); } return nullptr; } return HandleMemberPointerAccess(Info, BO->getLHS()->getType(), LV, BO->getRHS(), IncludeMember); } /// HandleBaseToDerivedCast - Apply the given base-to-derived cast operation on /// the provided lvalue, which currently refers to the base object. static bool HandleBaseToDerivedCast(EvalInfo &Info, const CastExpr *E, LValue &Result) { SubobjectDesignator &D = Result.Designator; if (D.Invalid || !Result.checkNullPointer(Info, E, CSK_Derived)) return false; QualType TargetQT = E->getType(); if (const PointerType *PT = TargetQT->getAs()) TargetQT = PT->getPointeeType(); // Check this cast lands within the final derived-to-base subobject path. if (D.MostDerivedPathLength + E->path_size() > D.Entries.size()) { Info.CCEDiag(E, diag::note_constexpr_invalid_downcast) << D.MostDerivedType << TargetQT; return false; } // Check the type of the final cast. We don't need to check the path, // since a cast can only be formed if the path is unique. unsigned NewEntriesSize = D.Entries.size() - E->path_size(); const CXXRecordDecl *TargetType = TargetQT->getAsCXXRecordDecl(); const CXXRecordDecl *FinalType; if (NewEntriesSize == D.MostDerivedPathLength) FinalType = D.MostDerivedType->getAsCXXRecordDecl(); else FinalType = getAsBaseClass(D.Entries[NewEntriesSize - 1]); if (FinalType->getCanonicalDecl() != TargetType->getCanonicalDecl()) { Info.CCEDiag(E, diag::note_constexpr_invalid_downcast) << D.MostDerivedType << TargetQT; return false; } // Truncate the lvalue to the appropriate derived class. return CastToDerivedClass(Info, E, Result, TargetType, NewEntriesSize); } namespace { enum EvalStmtResult { /// Evaluation failed. ESR_Failed, /// Hit a 'return' statement. ESR_Returned, /// Evaluation succeeded. ESR_Succeeded, /// Hit a 'continue' statement. ESR_Continue, /// Hit a 'break' statement. ESR_Break, /// Still scanning for 'case' or 'default' statement. ESR_CaseNotFound }; } static bool EvaluateVarDecl(EvalInfo &Info, const VarDecl *VD) { // We don't need to evaluate the initializer for a static local. if (!VD->hasLocalStorage()) return true; LValue Result; APValue &Val = createTemporary(VD, true, Result, *Info.CurrentCall); const Expr *InitE = VD->getInit(); if (!InitE) { Info.FFDiag(VD->getBeginLoc(), diag::note_constexpr_uninitialized) << false << VD->getType(); Val = APValue(); return false; } if (InitE->isValueDependent()) return false; if (!EvaluateInPlace(Val, Info, Result, InitE)) { // Wipe out any partially-computed value, to allow tracking that this // evaluation failed. Val = APValue(); return false; } return true; } static bool EvaluateDecl(EvalInfo &Info, const Decl *D) { bool OK = true; if (const VarDecl *VD = dyn_cast(D)) OK &= EvaluateVarDecl(Info, VD); if (const DecompositionDecl *DD = dyn_cast(D)) for (auto *BD : DD->bindings()) if (auto *VD = BD->getHoldingVar()) OK &= EvaluateDecl(Info, VD); return OK; } /// Evaluate a condition (either a variable declaration or an expression). static bool EvaluateCond(EvalInfo &Info, const VarDecl *CondDecl, const Expr *Cond, bool &Result) { FullExpressionRAII Scope(Info); if (CondDecl && !EvaluateDecl(Info, CondDecl)) return false; return EvaluateAsBooleanCondition(Cond, Result, Info); } namespace { /// A location where the result (returned value) of evaluating a /// statement should be stored. struct StmtResult { /// The APValue that should be filled in with the returned value. APValue &Value; /// The location containing the result, if any (used to support RVO). const LValue *Slot; }; struct TempVersionRAII { CallStackFrame &Frame; TempVersionRAII(CallStackFrame &Frame) : Frame(Frame) { Frame.pushTempVersion(); } ~TempVersionRAII() { Frame.popTempVersion(); } }; } static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info, const Stmt *S, const SwitchCase *SC = nullptr); /// Evaluate the body of a loop, and translate the result as appropriate. static EvalStmtResult EvaluateLoopBody(StmtResult &Result, EvalInfo &Info, const Stmt *Body, const SwitchCase *Case = nullptr) { BlockScopeRAII Scope(Info); switch (EvalStmtResult ESR = EvaluateStmt(Result, Info, Body, Case)) { case ESR_Break: return ESR_Succeeded; case ESR_Succeeded: case ESR_Continue: return ESR_Continue; case ESR_Failed: case ESR_Returned: case ESR_CaseNotFound: return ESR; } llvm_unreachable("Invalid EvalStmtResult!"); } /// Evaluate a switch statement. static EvalStmtResult EvaluateSwitch(StmtResult &Result, EvalInfo &Info, const SwitchStmt *SS) { BlockScopeRAII Scope(Info); // Evaluate the switch condition. APSInt Value; { FullExpressionRAII Scope(Info); if (const Stmt *Init = SS->getInit()) { EvalStmtResult ESR = EvaluateStmt(Result, Info, Init); if (ESR != ESR_Succeeded) return ESR; } if (SS->getConditionVariable() && !EvaluateDecl(Info, SS->getConditionVariable())) return ESR_Failed; if (!EvaluateInteger(SS->getCond(), Value, Info)) return ESR_Failed; } // Find the switch case corresponding to the value of the condition. // FIXME: Cache this lookup. const SwitchCase *Found = nullptr; for (const SwitchCase *SC = SS->getSwitchCaseList(); SC; SC = SC->getNextSwitchCase()) { if (isa(SC)) { Found = SC; continue; } const CaseStmt *CS = cast(SC); APSInt LHS = CS->getLHS()->EvaluateKnownConstInt(Info.Ctx); APSInt RHS = CS->getRHS() ? CS->getRHS()->EvaluateKnownConstInt(Info.Ctx) : LHS; if (LHS <= Value && Value <= RHS) { Found = SC; break; } } if (!Found) return ESR_Succeeded; // Search the switch body for the switch case and evaluate it from there. switch (EvalStmtResult ESR = EvaluateStmt(Result, Info, SS->getBody(), Found)) { case ESR_Break: return ESR_Succeeded; case ESR_Succeeded: case ESR_Continue: case ESR_Failed: case ESR_Returned: return ESR; case ESR_CaseNotFound: // This can only happen if the switch case is nested within a statement // expression. We have no intention of supporting that. Info.FFDiag(Found->getBeginLoc(), diag::note_constexpr_stmt_expr_unsupported); return ESR_Failed; } llvm_unreachable("Invalid EvalStmtResult!"); } // Evaluate a statement. static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info, const Stmt *S, const SwitchCase *Case) { if (!Info.nextStep(S)) return ESR_Failed; // If we're hunting down a 'case' or 'default' label, recurse through // substatements until we hit the label. if (Case) { // FIXME: We don't start the lifetime of objects whose initialization we // jump over. However, such objects must be of class type with a trivial // default constructor that initialize all subobjects, so must be empty, // so this almost never matters. switch (S->getStmtClass()) { case Stmt::CompoundStmtClass: // FIXME: Precompute which substatement of a compound statement we // would jump to, and go straight there rather than performing a // linear scan each time. case Stmt::LabelStmtClass: case Stmt::AttributedStmtClass: case Stmt::DoStmtClass: break; case Stmt::CaseStmtClass: case Stmt::DefaultStmtClass: if (Case == S) Case = nullptr; break; case Stmt::IfStmtClass: { // FIXME: Precompute which side of an 'if' we would jump to, and go // straight there rather than scanning both sides. const IfStmt *IS = cast(S); // Wrap the evaluation in a block scope, in case it's a DeclStmt // preceded by our switch label. BlockScopeRAII Scope(Info); EvalStmtResult ESR = EvaluateStmt(Result, Info, IS->getThen(), Case); if (ESR != ESR_CaseNotFound || !IS->getElse()) return ESR; return EvaluateStmt(Result, Info, IS->getElse(), Case); } case Stmt::WhileStmtClass: { EvalStmtResult ESR = EvaluateLoopBody(Result, Info, cast(S)->getBody(), Case); if (ESR != ESR_Continue) return ESR; break; } case Stmt::ForStmtClass: { const ForStmt *FS = cast(S); EvalStmtResult ESR = EvaluateLoopBody(Result, Info, FS->getBody(), Case); if (ESR != ESR_Continue) return ESR; if (FS->getInc()) { FullExpressionRAII IncScope(Info); if (!EvaluateIgnoredValue(Info, FS->getInc())) return ESR_Failed; } break; } case Stmt::DeclStmtClass: // FIXME: If the variable has initialization that can't be jumped over, // bail out of any immediately-surrounding compound-statement too. default: return ESR_CaseNotFound; } } switch (S->getStmtClass()) { default: if (const Expr *E = dyn_cast(S)) { // Don't bother evaluating beyond an expression-statement which couldn't // be evaluated. FullExpressionRAII Scope(Info); if (!EvaluateIgnoredValue(Info, E)) return ESR_Failed; return ESR_Succeeded; } Info.FFDiag(S->getBeginLoc()); return ESR_Failed; case Stmt::NullStmtClass: return ESR_Succeeded; case Stmt::DeclStmtClass: { const DeclStmt *DS = cast(S); for (const auto *DclIt : DS->decls()) { // Each declaration initialization is its own full-expression. // FIXME: This isn't quite right; if we're performing aggregate // initialization, each braced subexpression is its own full-expression. FullExpressionRAII Scope(Info); if (!EvaluateDecl(Info, DclIt) && !Info.noteFailure()) return ESR_Failed; } return ESR_Succeeded; } case Stmt::ReturnStmtClass: { const Expr *RetExpr = cast(S)->getRetValue(); FullExpressionRAII Scope(Info); if (RetExpr && !(Result.Slot ? EvaluateInPlace(Result.Value, Info, *Result.Slot, RetExpr) : Evaluate(Result.Value, Info, RetExpr))) return ESR_Failed; return ESR_Returned; } case Stmt::CompoundStmtClass: { BlockScopeRAII Scope(Info); const CompoundStmt *CS = cast(S); for (const auto *BI : CS->body()) { EvalStmtResult ESR = EvaluateStmt(Result, Info, BI, Case); if (ESR == ESR_Succeeded) Case = nullptr; else if (ESR != ESR_CaseNotFound) return ESR; } return Case ? ESR_CaseNotFound : ESR_Succeeded; } case Stmt::IfStmtClass: { const IfStmt *IS = cast(S); // Evaluate the condition, as either a var decl or as an expression. BlockScopeRAII Scope(Info); if (const Stmt *Init = IS->getInit()) { EvalStmtResult ESR = EvaluateStmt(Result, Info, Init); if (ESR != ESR_Succeeded) return ESR; } bool Cond; if (!EvaluateCond(Info, IS->getConditionVariable(), IS->getCond(), Cond)) return ESR_Failed; if (const Stmt *SubStmt = Cond ? IS->getThen() : IS->getElse()) { EvalStmtResult ESR = EvaluateStmt(Result, Info, SubStmt); if (ESR != ESR_Succeeded) return ESR; } return ESR_Succeeded; } case Stmt::WhileStmtClass: { const WhileStmt *WS = cast(S); while (true) { BlockScopeRAII Scope(Info); bool Continue; if (!EvaluateCond(Info, WS->getConditionVariable(), WS->getCond(), Continue)) return ESR_Failed; if (!Continue) break; EvalStmtResult ESR = EvaluateLoopBody(Result, Info, WS->getBody()); if (ESR != ESR_Continue) return ESR; } return ESR_Succeeded; } case Stmt::DoStmtClass: { const DoStmt *DS = cast(S); bool Continue; do { EvalStmtResult ESR = EvaluateLoopBody(Result, Info, DS->getBody(), Case); if (ESR != ESR_Continue) return ESR; Case = nullptr; FullExpressionRAII CondScope(Info); if (!EvaluateAsBooleanCondition(DS->getCond(), Continue, Info)) return ESR_Failed; } while (Continue); return ESR_Succeeded; } case Stmt::ForStmtClass: { const ForStmt *FS = cast(S); BlockScopeRAII Scope(Info); if (FS->getInit()) { EvalStmtResult ESR = EvaluateStmt(Result, Info, FS->getInit()); if (ESR != ESR_Succeeded) return ESR; } while (true) { BlockScopeRAII Scope(Info); bool Continue = true; if (FS->getCond() && !EvaluateCond(Info, FS->getConditionVariable(), FS->getCond(), Continue)) return ESR_Failed; if (!Continue) break; EvalStmtResult ESR = EvaluateLoopBody(Result, Info, FS->getBody()); if (ESR != ESR_Continue) return ESR; if (FS->getInc()) { FullExpressionRAII IncScope(Info); if (!EvaluateIgnoredValue(Info, FS->getInc())) return ESR_Failed; } } return ESR_Succeeded; } case Stmt::CXXForRangeStmtClass: { const CXXForRangeStmt *FS = cast(S); BlockScopeRAII Scope(Info); // Evaluate the init-statement if present. if (FS->getInit()) { EvalStmtResult ESR = EvaluateStmt(Result, Info, FS->getInit()); if (ESR != ESR_Succeeded) return ESR; } // Initialize the __range variable. EvalStmtResult ESR = EvaluateStmt(Result, Info, FS->getRangeStmt()); if (ESR != ESR_Succeeded) return ESR; // Create the __begin and __end iterators. ESR = EvaluateStmt(Result, Info, FS->getBeginStmt()); if (ESR != ESR_Succeeded) return ESR; ESR = EvaluateStmt(Result, Info, FS->getEndStmt()); if (ESR != ESR_Succeeded) return ESR; while (true) { // Condition: __begin != __end. { bool Continue = true; FullExpressionRAII CondExpr(Info); if (!EvaluateAsBooleanCondition(FS->getCond(), Continue, Info)) return ESR_Failed; if (!Continue) break; } // User's variable declaration, initialized by *__begin. BlockScopeRAII InnerScope(Info); ESR = EvaluateStmt(Result, Info, FS->getLoopVarStmt()); if (ESR != ESR_Succeeded) return ESR; // Loop body. ESR = EvaluateLoopBody(Result, Info, FS->getBody()); if (ESR != ESR_Continue) return ESR; // Increment: ++__begin if (!EvaluateIgnoredValue(Info, FS->getInc())) return ESR_Failed; } return ESR_Succeeded; } case Stmt::SwitchStmtClass: return EvaluateSwitch(Result, Info, cast(S)); case Stmt::ContinueStmtClass: return ESR_Continue; case Stmt::BreakStmtClass: return ESR_Break; case Stmt::LabelStmtClass: return EvaluateStmt(Result, Info, cast(S)->getSubStmt(), Case); case Stmt::AttributedStmtClass: // As a general principle, C++11 attributes can be ignored without // any semantic impact. return EvaluateStmt(Result, Info, cast(S)->getSubStmt(), Case); case Stmt::CaseStmtClass: case Stmt::DefaultStmtClass: return EvaluateStmt(Result, Info, cast(S)->getSubStmt(), Case); case Stmt::CXXTryStmtClass: // Evaluate try blocks by evaluating all sub statements. return EvaluateStmt(Result, Info, cast(S)->getTryBlock(), Case); } } /// CheckTrivialDefaultConstructor - Check whether a constructor is a trivial /// default constructor. If so, we'll fold it whether or not it's marked as /// constexpr. If it is marked as constexpr, we will never implicitly define it, /// so we need special handling. static bool CheckTrivialDefaultConstructor(EvalInfo &Info, SourceLocation Loc, const CXXConstructorDecl *CD, bool IsValueInitialization) { if (!CD->isTrivial() || !CD->isDefaultConstructor()) return false; // Value-initialization does not call a trivial default constructor, so such a // call is a core constant expression whether or not the constructor is // constexpr. if (!CD->isConstexpr() && !IsValueInitialization) { if (Info.getLangOpts().CPlusPlus11) { // FIXME: If DiagDecl is an implicitly-declared special member function, // we should be much more explicit about why it's not constexpr. Info.CCEDiag(Loc, diag::note_constexpr_invalid_function, 1) << /*IsConstexpr*/0 << /*IsConstructor*/1 << CD; Info.Note(CD->getLocation(), diag::note_declared_at); } else { Info.CCEDiag(Loc, diag::note_invalid_subexpr_in_const_expr); } } return true; } /// CheckConstexprFunction - Check that a function can be called in a constant /// expression. static bool CheckConstexprFunction(EvalInfo &Info, SourceLocation CallLoc, const FunctionDecl *Declaration, const FunctionDecl *Definition, const Stmt *Body) { // Potential constant expressions can contain calls to declared, but not yet // defined, constexpr functions. if (Info.checkingPotentialConstantExpression() && !Definition && Declaration->isConstexpr()) return false; // Bail out if the function declaration itself is invalid. We will // have produced a relevant diagnostic while parsing it, so just // note the problematic sub-expression. if (Declaration->isInvalidDecl()) { Info.FFDiag(CallLoc, diag::note_invalid_subexpr_in_const_expr); return false; } // DR1872: An instantiated virtual constexpr function can't be called in a // constant expression (prior to C++20). We can still constant-fold such a // call. if (!Info.Ctx.getLangOpts().CPlusPlus2a && isa(Declaration) && cast(Declaration)->isVirtual()) Info.CCEDiag(CallLoc, diag::note_constexpr_virtual_call); if (Definition && Definition->isInvalidDecl()) { Info.FFDiag(CallLoc, diag::note_invalid_subexpr_in_const_expr); return false; } // Can we evaluate this function call? if (Definition && Definition->isConstexpr() && Body) return true; if (Info.getLangOpts().CPlusPlus11) { const FunctionDecl *DiagDecl = Definition ? Definition : Declaration; // If this function is not constexpr because it is an inherited // non-constexpr constructor, diagnose that directly. auto *CD = dyn_cast(DiagDecl); if (CD && CD->isInheritingConstructor()) { auto *Inherited = CD->getInheritedConstructor().getConstructor(); if (!Inherited->isConstexpr()) DiagDecl = CD = Inherited; } // FIXME: If DiagDecl is an implicitly-declared special member function // or an inheriting constructor, we should be much more explicit about why // it's not constexpr. if (CD && CD->isInheritingConstructor()) Info.FFDiag(CallLoc, diag::note_constexpr_invalid_inhctor, 1) << CD->getInheritedConstructor().getConstructor()->getParent(); else Info.FFDiag(CallLoc, diag::note_constexpr_invalid_function, 1) << DiagDecl->isConstexpr() << (bool)CD << DiagDecl; Info.Note(DiagDecl->getLocation(), diag::note_declared_at); } else { Info.FFDiag(CallLoc, diag::note_invalid_subexpr_in_const_expr); } return false; } namespace { struct CheckDynamicTypeHandler { AccessKinds AccessKind; typedef bool result_type; bool failed() { return false; } bool found(APValue &Subobj, QualType SubobjType) { return true; } bool found(APSInt &Value, QualType SubobjType) { return true; } bool found(APFloat &Value, QualType SubobjType) { return true; } }; } // end anonymous namespace /// Check that we can access the notional vptr of an object / determine its /// dynamic type. static bool checkDynamicType(EvalInfo &Info, const Expr *E, const LValue &This, AccessKinds AK, bool Polymorphic) { if (This.Designator.Invalid) return false; CompleteObject Obj = findCompleteObject(Info, E, AK, This, QualType()); if (!Obj) return false; if (!Obj.Value) { // The object is not usable in constant expressions, so we can't inspect // its value to see if it's in-lifetime or what the active union members // are. We can still check for a one-past-the-end lvalue. if (This.Designator.isOnePastTheEnd() || This.Designator.isMostDerivedAnUnsizedArray()) { Info.FFDiag(E, This.Designator.isOnePastTheEnd() ? diag::note_constexpr_access_past_end : diag::note_constexpr_access_unsized_array) << AK; return false; } else if (Polymorphic) { // Conservatively refuse to perform a polymorphic operation if we would // not be able to read a notional 'vptr' value. APValue Val; This.moveInto(Val); QualType StarThisType = Info.Ctx.getLValueReferenceType(This.Designator.getType(Info.Ctx)); Info.FFDiag(E, diag::note_constexpr_polymorphic_unknown_dynamic_type) << AK << Val.getAsString(Info.Ctx, StarThisType); return false; } return true; } CheckDynamicTypeHandler Handler{AK}; return Obj && findSubobject(Info, E, Obj, This.Designator, Handler); } /// Check that the pointee of the 'this' pointer in a member function call is /// either within its lifetime or in its period of construction or destruction. static bool checkNonVirtualMemberCallThisPointer(EvalInfo &Info, const Expr *E, const LValue &This) { return checkDynamicType(Info, E, This, AK_MemberCall, false); } struct DynamicType { /// The dynamic class type of the object. const CXXRecordDecl *Type; /// The corresponding path length in the lvalue. unsigned PathLength; }; static const CXXRecordDecl *getBaseClassType(SubobjectDesignator &Designator, unsigned PathLength) { assert(PathLength >= Designator.MostDerivedPathLength && PathLength <= Designator.Entries.size() && "invalid path length"); return (PathLength == Designator.MostDerivedPathLength) ? Designator.MostDerivedType->getAsCXXRecordDecl() : getAsBaseClass(Designator.Entries[PathLength - 1]); } /// Determine the dynamic type of an object. static Optional ComputeDynamicType(EvalInfo &Info, const Expr *E, LValue &This, AccessKinds AK) { // If we don't have an lvalue denoting an object of class type, there is no // meaningful dynamic type. (We consider objects of non-class type to have no // dynamic type.) if (!checkDynamicType(Info, E, This, AK, true)) return None; // Refuse to compute a dynamic type in the presence of virtual bases. This // shouldn't happen other than in constant-folding situations, since literal // types can't have virtual bases. // // Note that consumers of DynamicType assume that the type has no virtual // bases, and will need modifications if this restriction is relaxed. const CXXRecordDecl *Class = This.Designator.MostDerivedType->getAsCXXRecordDecl(); if (!Class || Class->getNumVBases()) { Info.FFDiag(E); return None; } // FIXME: For very deep class hierarchies, it might be beneficial to use a // binary search here instead. But the overwhelmingly common case is that // we're not in the middle of a constructor, so it probably doesn't matter // in practice. ArrayRef Path = This.Designator.Entries; for (unsigned PathLength = This.Designator.MostDerivedPathLength; PathLength <= Path.size(); ++PathLength) { switch (Info.isEvaluatingConstructor(This.getLValueBase(), Path.slice(0, PathLength))) { case ConstructionPhase::Bases: // We're constructing a base class. This is not the dynamic type. break; case ConstructionPhase::None: case ConstructionPhase::AfterBases: // We've finished constructing the base classes, so this is the dynamic // type. return DynamicType{getBaseClassType(This.Designator, PathLength), PathLength}; } } // CWG issue 1517: we're constructing a base class of the object described by // 'This', so that object has not yet begun its period of construction and // any polymorphic operation on it results in undefined behavior. Info.FFDiag(E); return None; } /// Perform virtual dispatch. static const CXXMethodDecl *HandleVirtualDispatch( EvalInfo &Info, const Expr *E, LValue &This, const CXXMethodDecl *Found, llvm::SmallVectorImpl &CovariantAdjustmentPath) { Optional DynType = ComputeDynamicType(Info, E, This, AK_MemberCall); if (!DynType) return nullptr; // Find the final overrider. It must be declared in one of the classes on the // path from the dynamic type to the static type. // FIXME: If we ever allow literal types to have virtual base classes, that // won't be true. const CXXMethodDecl *Callee = Found; unsigned PathLength = DynType->PathLength; for (/**/; PathLength <= This.Designator.Entries.size(); ++PathLength) { const CXXRecordDecl *Class = getBaseClassType(This.Designator, PathLength); const CXXMethodDecl *Overrider = Found->getCorrespondingMethodDeclaredInClass(Class, false); if (Overrider) { Callee = Overrider; break; } } // C++2a [class.abstract]p6: // the effect of making a virtual call to a pure virtual function [...] is // undefined if (Callee->isPure()) { Info.FFDiag(E, diag::note_constexpr_pure_virtual_call, 1) << Callee; Info.Note(Callee->getLocation(), diag::note_declared_at); return nullptr; } // If necessary, walk the rest of the path to determine the sequence of // covariant adjustment steps to apply. if (!Info.Ctx.hasSameUnqualifiedType(Callee->getReturnType(), Found->getReturnType())) { CovariantAdjustmentPath.push_back(Callee->getReturnType()); for (unsigned CovariantPathLength = PathLength + 1; CovariantPathLength != This.Designator.Entries.size(); ++CovariantPathLength) { const CXXRecordDecl *NextClass = getBaseClassType(This.Designator, CovariantPathLength); const CXXMethodDecl *Next = Found->getCorrespondingMethodDeclaredInClass(NextClass, false); if (Next && !Info.Ctx.hasSameUnqualifiedType( Next->getReturnType(), CovariantAdjustmentPath.back())) CovariantAdjustmentPath.push_back(Next->getReturnType()); } if (!Info.Ctx.hasSameUnqualifiedType(Found->getReturnType(), CovariantAdjustmentPath.back())) CovariantAdjustmentPath.push_back(Found->getReturnType()); } // Perform 'this' adjustment. if (!CastToDerivedClass(Info, E, This, Callee->getParent(), PathLength)) return nullptr; return Callee; } /// Perform the adjustment from a value returned by a virtual function to /// a value of the statically expected type, which may be a pointer or /// reference to a base class of the returned type. static bool HandleCovariantReturnAdjustment(EvalInfo &Info, const Expr *E, APValue &Result, ArrayRef Path) { assert(Result.isLValue() && "unexpected kind of APValue for covariant return"); if (Result.isNullPointer()) return true; LValue LVal; LVal.setFrom(Info.Ctx, Result); const CXXRecordDecl *OldClass = Path[0]->getPointeeCXXRecordDecl(); for (unsigned I = 1; I != Path.size(); ++I) { const CXXRecordDecl *NewClass = Path[I]->getPointeeCXXRecordDecl(); assert(OldClass && NewClass && "unexpected kind of covariant return"); if (OldClass != NewClass && !CastToBaseClass(Info, E, LVal, OldClass, NewClass)) return false; OldClass = NewClass; } LVal.moveInto(Result); return true; } /// Determine whether \p Base, which is known to be a direct base class of /// \p Derived, is a public base class. static bool isBaseClassPublic(const CXXRecordDecl *Derived, const CXXRecordDecl *Base) { for (const CXXBaseSpecifier &BaseSpec : Derived->bases()) { auto *BaseClass = BaseSpec.getType()->getAsCXXRecordDecl(); if (BaseClass && declaresSameEntity(BaseClass, Base)) return BaseSpec.getAccessSpecifier() == AS_public; } llvm_unreachable("Base is not a direct base of Derived"); } /// Apply the given dynamic cast operation on the provided lvalue. /// /// This implements the hard case of dynamic_cast, requiring a "runtime check" /// to find a suitable target subobject. static bool HandleDynamicCast(EvalInfo &Info, const ExplicitCastExpr *E, LValue &Ptr) { // We can't do anything with a non-symbolic pointer value. SubobjectDesignator &D = Ptr.Designator; if (D.Invalid) return false; // C++ [expr.dynamic.cast]p6: // If v is a null pointer value, the result is a null pointer value. if (Ptr.isNullPointer() && !E->isGLValue()) return true; // For all the other cases, we need the pointer to point to an object within // its lifetime / period of construction / destruction, and we need to know // its dynamic type. Optional DynType = ComputeDynamicType(Info, E, Ptr, AK_DynamicCast); if (!DynType) return false; // C++ [expr.dynamic.cast]p7: // If T is "pointer to cv void", then the result is a pointer to the most // derived object if (E->getType()->isVoidPointerType()) return CastToDerivedClass(Info, E, Ptr, DynType->Type, DynType->PathLength); const CXXRecordDecl *C = E->getTypeAsWritten()->getPointeeCXXRecordDecl(); assert(C && "dynamic_cast target is not void pointer nor class"); CanQualType CQT = Info.Ctx.getCanonicalType(Info.Ctx.getRecordType(C)); auto RuntimeCheckFailed = [&] (CXXBasePaths *Paths) { // C++ [expr.dynamic.cast]p9: if (!E->isGLValue()) { // The value of a failed cast to pointer type is the null pointer value // of the required result type. auto TargetVal = Info.Ctx.getTargetNullPointerValue(E->getType()); Ptr.setNull(E->getType(), TargetVal); return true; } // A failed cast to reference type throws [...] std::bad_cast. unsigned DiagKind; if (!Paths && (declaresSameEntity(DynType->Type, C) || DynType->Type->isDerivedFrom(C))) DiagKind = 0; else if (!Paths || Paths->begin() == Paths->end()) DiagKind = 1; else if (Paths->isAmbiguous(CQT)) DiagKind = 2; else { assert(Paths->front().Access != AS_public && "why did the cast fail?"); DiagKind = 3; } Info.FFDiag(E, diag::note_constexpr_dynamic_cast_to_reference_failed) << DiagKind << Ptr.Designator.getType(Info.Ctx) << Info.Ctx.getRecordType(DynType->Type) << E->getType().getUnqualifiedType(); return false; }; // Runtime check, phase 1: // Walk from the base subobject towards the derived object looking for the // target type. for (int PathLength = Ptr.Designator.Entries.size(); PathLength >= (int)DynType->PathLength; --PathLength) { const CXXRecordDecl *Class = getBaseClassType(Ptr.Designator, PathLength); if (declaresSameEntity(Class, C)) return CastToDerivedClass(Info, E, Ptr, Class, PathLength); // We can only walk across public inheritance edges. if (PathLength > (int)DynType->PathLength && !isBaseClassPublic(getBaseClassType(Ptr.Designator, PathLength - 1), Class)) return RuntimeCheckFailed(nullptr); } // Runtime check, phase 2: // Search the dynamic type for an unambiguous public base of type C. CXXBasePaths Paths(/*FindAmbiguities=*/true, /*RecordPaths=*/true, /*DetectVirtual=*/false); if (DynType->Type->isDerivedFrom(C, Paths) && !Paths.isAmbiguous(CQT) && Paths.front().Access == AS_public) { // Downcast to the dynamic type... if (!CastToDerivedClass(Info, E, Ptr, DynType->Type, DynType->PathLength)) return false; // ... then upcast to the chosen base class subobject. for (CXXBasePathElement &Elem : Paths.front()) if (!HandleLValueBase(Info, E, Ptr, Elem.Class, Elem.Base)) return false; return true; } // Otherwise, the runtime check fails. return RuntimeCheckFailed(&Paths); } namespace { struct StartLifetimeOfUnionMemberHandler { const FieldDecl *Field; static const AccessKinds AccessKind = AK_Assign; APValue getDefaultInitValue(QualType SubobjType) { if (auto *RD = SubobjType->getAsCXXRecordDecl()) { if (RD->isUnion()) return APValue((const FieldDecl*)nullptr); APValue Struct(APValue::UninitStruct(), RD->getNumBases(), std::distance(RD->field_begin(), RD->field_end())); unsigned Index = 0; for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(), End = RD->bases_end(); I != End; ++I, ++Index) Struct.getStructBase(Index) = getDefaultInitValue(I->getType()); for (const auto *I : RD->fields()) { if (I->isUnnamedBitfield()) continue; Struct.getStructField(I->getFieldIndex()) = getDefaultInitValue(I->getType()); } return Struct; } if (auto *AT = dyn_cast_or_null( SubobjType->getAsArrayTypeUnsafe())) { APValue Array(APValue::UninitArray(), 0, AT->getSize().getZExtValue()); if (Array.hasArrayFiller()) Array.getArrayFiller() = getDefaultInitValue(AT->getElementType()); return Array; } return APValue::IndeterminateValue(); } typedef bool result_type; bool failed() { return false; } bool found(APValue &Subobj, QualType SubobjType) { // We are supposed to perform no initialization but begin the lifetime of // the object. We interpret that as meaning to do what default // initialization of the object would do if all constructors involved were // trivial: // * All base, non-variant member, and array element subobjects' lifetimes // begin // * No variant members' lifetimes begin // * All scalar subobjects whose lifetimes begin have indeterminate values assert(SubobjType->isUnionType()); if (!declaresSameEntity(Subobj.getUnionField(), Field)) Subobj.setUnion(Field, getDefaultInitValue(Field->getType())); return true; } bool found(APSInt &Value, QualType SubobjType) { llvm_unreachable("wrong value kind for union object"); } bool found(APFloat &Value, QualType SubobjType) { llvm_unreachable("wrong value kind for union object"); } }; } // end anonymous namespace const AccessKinds StartLifetimeOfUnionMemberHandler::AccessKind; /// Handle a builtin simple-assignment or a call to a trivial assignment /// operator whose left-hand side might involve a union member access. If it /// does, implicitly start the lifetime of any accessed union elements per /// C++20 [class.union]5. static bool HandleUnionActiveMemberChange(EvalInfo &Info, const Expr *LHSExpr, const LValue &LHS) { if (LHS.InvalidBase || LHS.Designator.Invalid) return false; llvm::SmallVector, 4> UnionPathLengths; // C++ [class.union]p5: // define the set S(E) of subexpressions of E as follows: unsigned PathLength = LHS.Designator.Entries.size(); for (const Expr *E = LHSExpr; E != nullptr;) { // -- If E is of the form A.B, S(E) contains the elements of S(A)... if (auto *ME = dyn_cast(E)) { auto *FD = dyn_cast(ME->getMemberDecl()); if (!FD) break; // ... and also contains A.B if B names a union member if (FD->getParent()->isUnion()) UnionPathLengths.push_back({PathLength - 1, FD}); E = ME->getBase(); --PathLength; assert(declaresSameEntity(FD, LHS.Designator.Entries[PathLength] .getAsBaseOrMember().getPointer())); // -- If E is of the form A[B] and is interpreted as a built-in array // subscripting operator, S(E) is [S(the array operand, if any)]. } else if (auto *ASE = dyn_cast(E)) { // Step over an ArrayToPointerDecay implicit cast. auto *Base = ASE->getBase()->IgnoreImplicit(); if (!Base->getType()->isArrayType()) break; E = Base; --PathLength; } else if (auto *ICE = dyn_cast(E)) { // Step over a derived-to-base conversion. E = ICE->getSubExpr(); if (ICE->getCastKind() == CK_NoOp) continue; if (ICE->getCastKind() != CK_DerivedToBase && ICE->getCastKind() != CK_UncheckedDerivedToBase) break; // Walk path backwards as we walk up from the base to the derived class. for (const CXXBaseSpecifier *Elt : llvm::reverse(ICE->path())) { --PathLength; (void)Elt; assert(declaresSameEntity(Elt->getType()->getAsCXXRecordDecl(), LHS.Designator.Entries[PathLength] .getAsBaseOrMember().getPointer())); } // -- Otherwise, S(E) is empty. } else { break; } } // Common case: no unions' lifetimes are started. if (UnionPathLengths.empty()) return true; // if modification of X [would access an inactive union member], an object // of the type of X is implicitly created CompleteObject Obj = findCompleteObject(Info, LHSExpr, AK_Assign, LHS, LHSExpr->getType()); if (!Obj) return false; for (std::pair LengthAndField : llvm::reverse(UnionPathLengths)) { // Form a designator for the union object. SubobjectDesignator D = LHS.Designator; D.truncate(Info.Ctx, LHS.Base, LengthAndField.first); StartLifetimeOfUnionMemberHandler StartLifetime{LengthAndField.second}; if (!findSubobject(Info, LHSExpr, Obj, D, StartLifetime)) return false; } return true; } /// Determine if a class has any fields that might need to be copied by a /// trivial copy or move operation. static bool hasFields(const CXXRecordDecl *RD) { if (!RD || RD->isEmpty()) return false; for (auto *FD : RD->fields()) { if (FD->isUnnamedBitfield()) continue; return true; } for (auto &Base : RD->bases()) if (hasFields(Base.getType()->getAsCXXRecordDecl())) return true; return false; } namespace { typedef SmallVector ArgVector; } /// EvaluateArgs - Evaluate the arguments to a function call. static bool EvaluateArgs(ArrayRef Args, ArgVector &ArgValues, EvalInfo &Info, const FunctionDecl *Callee) { bool Success = true; llvm::SmallBitVector ForbiddenNullArgs; if (Callee->hasAttr()) { ForbiddenNullArgs.resize(Args.size()); for (const auto *Attr : Callee->specific_attrs()) { if (!Attr->args_size()) { ForbiddenNullArgs.set(); break; } else for (auto Idx : Attr->args()) { unsigned ASTIdx = Idx.getASTIndex(); if (ASTIdx >= Args.size()) continue; ForbiddenNullArgs[ASTIdx] = 1; } } } for (ArrayRef::iterator I = Args.begin(), E = Args.end(); I != E; ++I) { if (!Evaluate(ArgValues[I - Args.begin()], Info, *I)) { // If we're checking for a potential constant expression, evaluate all // initializers even if some of them fail. if (!Info.noteFailure()) return false; Success = false; } else if (!ForbiddenNullArgs.empty() && ForbiddenNullArgs[I - Args.begin()] && ArgValues[I - Args.begin()].isNullPointer()) { Info.CCEDiag(*I, diag::note_non_null_attribute_failed); if (!Info.noteFailure()) return false; Success = false; } } return Success; } /// Evaluate a function call. static bool HandleFunctionCall(SourceLocation CallLoc, const FunctionDecl *Callee, const LValue *This, ArrayRef Args, const Stmt *Body, EvalInfo &Info, APValue &Result, const LValue *ResultSlot) { ArgVector ArgValues(Args.size()); if (!EvaluateArgs(Args, ArgValues, Info, Callee)) return false; if (!Info.CheckCallLimit(CallLoc)) return false; CallStackFrame Frame(Info, CallLoc, Callee, This, ArgValues.data()); // For a trivial copy or move assignment, perform an APValue copy. This is // essential for unions, where the operations performed by the assignment // operator cannot be represented as statements. // // Skip this for non-union classes with no fields; in that case, the defaulted // copy/move does not actually read the object. const CXXMethodDecl *MD = dyn_cast(Callee); if (MD && MD->isDefaulted() && (MD->getParent()->isUnion() || (MD->isTrivial() && hasFields(MD->getParent())))) { assert(This && (MD->isCopyAssignmentOperator() || MD->isMoveAssignmentOperator())); LValue RHS; RHS.setFrom(Info.Ctx, ArgValues[0]); APValue RHSValue; if (!handleLValueToRValueConversion(Info, Args[0], Args[0]->getType(), RHS, RHSValue)) return false; if (Info.getLangOpts().CPlusPlus2a && MD->isTrivial() && !HandleUnionActiveMemberChange(Info, Args[0], *This)) return false; if (!handleAssignment(Info, Args[0], *This, MD->getThisType(), RHSValue)) return false; This->moveInto(Result); return true; } else if (MD && isLambdaCallOperator(MD)) { // We're in a lambda; determine the lambda capture field maps unless we're // just constexpr checking a lambda's call operator. constexpr checking is // done before the captures have been added to the closure object (unless // we're inferring constexpr-ness), so we don't have access to them in this // case. But since we don't need the captures to constexpr check, we can // just ignore them. if (!Info.checkingPotentialConstantExpression()) MD->getParent()->getCaptureFields(Frame.LambdaCaptureFields, Frame.LambdaThisCaptureField); } StmtResult Ret = {Result, ResultSlot}; EvalStmtResult ESR = EvaluateStmt(Ret, Info, Body); if (ESR == ESR_Succeeded) { if (Callee->getReturnType()->isVoidType()) return true; Info.FFDiag(Callee->getEndLoc(), diag::note_constexpr_no_return); } return ESR == ESR_Returned; } /// Evaluate a constructor call. static bool HandleConstructorCall(const Expr *E, const LValue &This, APValue *ArgValues, const CXXConstructorDecl *Definition, EvalInfo &Info, APValue &Result) { SourceLocation CallLoc = E->getExprLoc(); if (!Info.CheckCallLimit(CallLoc)) return false; const CXXRecordDecl *RD = Definition->getParent(); if (RD->getNumVBases()) { Info.FFDiag(CallLoc, diag::note_constexpr_virtual_base) << RD; return false; } EvalInfo::EvaluatingConstructorRAII EvalObj( Info, ObjectUnderConstruction{This.getLValueBase(), This.Designator.Entries}, RD->getNumBases()); CallStackFrame Frame(Info, CallLoc, Definition, &This, ArgValues); // FIXME: Creating an APValue just to hold a nonexistent return value is // wasteful. APValue RetVal; StmtResult Ret = {RetVal, nullptr}; // If it's a delegating constructor, delegate. if (Definition->isDelegatingConstructor()) { CXXConstructorDecl::init_const_iterator I = Definition->init_begin(); { FullExpressionRAII InitScope(Info); if (!EvaluateInPlace(Result, Info, This, (*I)->getInit())) return false; } return EvaluateStmt(Ret, Info, Definition->getBody()) != ESR_Failed; } // For a trivial copy or move constructor, perform an APValue copy. This is // essential for unions (or classes with anonymous union members), where the // operations performed by the constructor cannot be represented by // ctor-initializers. // // Skip this for empty non-union classes; we should not perform an // lvalue-to-rvalue conversion on them because their copy constructor does not // actually read them. if (Definition->isDefaulted() && Definition->isCopyOrMoveConstructor() && (Definition->getParent()->isUnion() || (Definition->isTrivial() && hasFields(Definition->getParent())))) { LValue RHS; RHS.setFrom(Info.Ctx, ArgValues[0]); return handleLValueToRValueConversion( Info, E, Definition->getParamDecl(0)->getType().getNonReferenceType(), RHS, Result); } // Reserve space for the struct members. if (!RD->isUnion() && !Result.hasValue()) Result = APValue(APValue::UninitStruct(), RD->getNumBases(), std::distance(RD->field_begin(), RD->field_end())); if (RD->isInvalidDecl()) return false; const ASTRecordLayout &Layout = Info.Ctx.getASTRecordLayout(RD); // A scope for temporaries lifetime-extended by reference members. BlockScopeRAII LifetimeExtendedScope(Info); bool Success = true; unsigned BasesSeen = 0; #ifndef NDEBUG CXXRecordDecl::base_class_const_iterator BaseIt = RD->bases_begin(); #endif for (const auto *I : Definition->inits()) { LValue Subobject = This; LValue SubobjectParent = This; APValue *Value = &Result; // Determine the subobject to initialize. FieldDecl *FD = nullptr; if (I->isBaseInitializer()) { QualType BaseType(I->getBaseClass(), 0); #ifndef NDEBUG // Non-virtual base classes are initialized in the order in the class // definition. We have already checked for virtual base classes. assert(!BaseIt->isVirtual() && "virtual base for literal type"); assert(Info.Ctx.hasSameType(BaseIt->getType(), BaseType) && "base class initializers not in expected order"); ++BaseIt; #endif if (!HandleLValueDirectBase(Info, I->getInit(), Subobject, RD, BaseType->getAsCXXRecordDecl(), &Layout)) return false; Value = &Result.getStructBase(BasesSeen++); } else if ((FD = I->getMember())) { if (!HandleLValueMember(Info, I->getInit(), Subobject, FD, &Layout)) return false; if (RD->isUnion()) { Result = APValue(FD); Value = &Result.getUnionValue(); } else { Value = &Result.getStructField(FD->getFieldIndex()); } } else if (IndirectFieldDecl *IFD = I->getIndirectMember()) { // Walk the indirect field decl's chain to find the object to initialize, // and make sure we've initialized every step along it. auto IndirectFieldChain = IFD->chain(); for (auto *C : IndirectFieldChain) { FD = cast(C); CXXRecordDecl *CD = cast(FD->getParent()); // Switch the union field if it differs. This happens if we had // preceding zero-initialization, and we're now initializing a union // subobject other than the first. // FIXME: In this case, the values of the other subobjects are // specified, since zero-initialization sets all padding bits to zero. if (!Value->hasValue() || (Value->isUnion() && Value->getUnionField() != FD)) { if (CD->isUnion()) *Value = APValue(FD); else *Value = APValue(APValue::UninitStruct(), CD->getNumBases(), std::distance(CD->field_begin(), CD->field_end())); } // Store Subobject as its parent before updating it for the last element // in the chain. if (C == IndirectFieldChain.back()) SubobjectParent = Subobject; if (!HandleLValueMember(Info, I->getInit(), Subobject, FD)) return false; if (CD->isUnion()) Value = &Value->getUnionValue(); else Value = &Value->getStructField(FD->getFieldIndex()); } } else { llvm_unreachable("unknown base initializer kind"); } // Need to override This for implicit field initializers as in this case // This refers to innermost anonymous struct/union containing initializer, // not to currently constructed class. const Expr *Init = I->getInit(); ThisOverrideRAII ThisOverride(*Info.CurrentCall, &SubobjectParent, isa(Init)); FullExpressionRAII InitScope(Info); if (!EvaluateInPlace(*Value, Info, Subobject, Init) || (FD && FD->isBitField() && !truncateBitfieldValue(Info, Init, *Value, FD))) { // If we're checking for a potential constant expression, evaluate all // initializers even if some of them fail. if (!Info.noteFailure()) return false; Success = false; } // This is the point at which the dynamic type of the object becomes this // class type. if (I->isBaseInitializer() && BasesSeen == RD->getNumBases()) EvalObj.finishedConstructingBases(); } return Success && EvaluateStmt(Ret, Info, Definition->getBody()) != ESR_Failed; } static bool HandleConstructorCall(const Expr *E, const LValue &This, ArrayRef Args, const CXXConstructorDecl *Definition, EvalInfo &Info, APValue &Result) { ArgVector ArgValues(Args.size()); if (!EvaluateArgs(Args, ArgValues, Info, Definition)) return false; return HandleConstructorCall(E, This, ArgValues.data(), Definition, Info, Result); } //===----------------------------------------------------------------------===// // Generic Evaluation //===----------------------------------------------------------------------===// namespace { class BitCastBuffer { // FIXME: We're going to need bit-level granularity when we support // bit-fields. // FIXME: Its possible under the C++ standard for 'char' to not be 8 bits, but // we don't support a host or target where that is the case. Still, we should // use a more generic type in case we ever do. SmallVector, 32> Bytes; static_assert(std::numeric_limits::digits >= 8, "Need at least 8 bit unsigned char"); bool TargetIsLittleEndian; public: BitCastBuffer(CharUnits Width, bool TargetIsLittleEndian) : Bytes(Width.getQuantity()), TargetIsLittleEndian(TargetIsLittleEndian) {} LLVM_NODISCARD bool readObject(CharUnits Offset, CharUnits Width, SmallVectorImpl &Output) const { for (CharUnits I = Offset, E = Offset + Width; I != E; ++I) { // If a byte of an integer is uninitialized, then the whole integer is // uninitalized. if (!Bytes[I.getQuantity()]) return false; Output.push_back(*Bytes[I.getQuantity()]); } if (llvm::sys::IsLittleEndianHost != TargetIsLittleEndian) std::reverse(Output.begin(), Output.end()); return true; } void writeObject(CharUnits Offset, SmallVectorImpl &Input) { if (llvm::sys::IsLittleEndianHost != TargetIsLittleEndian) std::reverse(Input.begin(), Input.end()); size_t Index = 0; for (unsigned char Byte : Input) { assert(!Bytes[Offset.getQuantity() + Index] && "overwriting a byte?"); Bytes[Offset.getQuantity() + Index] = Byte; ++Index; } } size_t size() { return Bytes.size(); } }; /// Traverse an APValue to produce an BitCastBuffer, emulating how the current /// target would represent the value at runtime. class APValueToBufferConverter { EvalInfo &Info; BitCastBuffer Buffer; const CastExpr *BCE; APValueToBufferConverter(EvalInfo &Info, CharUnits ObjectWidth, const CastExpr *BCE) : Info(Info), Buffer(ObjectWidth, Info.Ctx.getTargetInfo().isLittleEndian()), BCE(BCE) {} bool visit(const APValue &Val, QualType Ty) { return visit(Val, Ty, CharUnits::fromQuantity(0)); } // Write out Val with type Ty into Buffer starting at Offset. bool visit(const APValue &Val, QualType Ty, CharUnits Offset) { assert((size_t)Offset.getQuantity() <= Buffer.size()); // As a special case, nullptr_t has an indeterminate value. if (Ty->isNullPtrType()) return true; // Dig through Src to find the byte at SrcOffset. switch (Val.getKind()) { case APValue::Indeterminate: case APValue::None: return true; case APValue::Int: return visitInt(Val.getInt(), Ty, Offset); case APValue::Float: return visitFloat(Val.getFloat(), Ty, Offset); case APValue::Array: return visitArray(Val, Ty, Offset); case APValue::Struct: return visitRecord(Val, Ty, Offset); case APValue::ComplexInt: case APValue::ComplexFloat: case APValue::Vector: case APValue::FixedPoint: // FIXME: We should support these. case APValue::Union: case APValue::MemberPointer: case APValue::AddrLabelDiff: { Info.FFDiag(BCE->getBeginLoc(), diag::note_constexpr_bit_cast_unsupported_type) << Ty; return false; } case APValue::LValue: llvm_unreachable("LValue subobject in bit_cast?"); } llvm_unreachable("Unhandled APValue::ValueKind"); } bool visitRecord(const APValue &Val, QualType Ty, CharUnits Offset) { const RecordDecl *RD = Ty->getAsRecordDecl(); const ASTRecordLayout &Layout = Info.Ctx.getASTRecordLayout(RD); // Visit the base classes. if (auto *CXXRD = dyn_cast(RD)) { for (size_t I = 0, E = CXXRD->getNumBases(); I != E; ++I) { const CXXBaseSpecifier &BS = CXXRD->bases_begin()[I]; CXXRecordDecl *BaseDecl = BS.getType()->getAsCXXRecordDecl(); if (!visitRecord(Val.getStructBase(I), BS.getType(), Layout.getBaseClassOffset(BaseDecl) + Offset)) return false; } } // Visit the fields. unsigned FieldIdx = 0; for (FieldDecl *FD : RD->fields()) { if (FD->isBitField()) { Info.FFDiag(BCE->getBeginLoc(), diag::note_constexpr_bit_cast_unsupported_bitfield); return false; } uint64_t FieldOffsetBits = Layout.getFieldOffset(FieldIdx); assert(FieldOffsetBits % Info.Ctx.getCharWidth() == 0 && "only bit-fields can have sub-char alignment"); CharUnits FieldOffset = Info.Ctx.toCharUnitsFromBits(FieldOffsetBits) + Offset; QualType FieldTy = FD->getType(); if (!visit(Val.getStructField(FieldIdx), FieldTy, FieldOffset)) return false; ++FieldIdx; } return true; } bool visitArray(const APValue &Val, QualType Ty, CharUnits Offset) { const auto *CAT = dyn_cast_or_null(Ty->getAsArrayTypeUnsafe()); if (!CAT) return false; CharUnits ElemWidth = Info.Ctx.getTypeSizeInChars(CAT->getElementType()); unsigned NumInitializedElts = Val.getArrayInitializedElts(); unsigned ArraySize = Val.getArraySize(); // First, initialize the initialized elements. for (unsigned I = 0; I != NumInitializedElts; ++I) { const APValue &SubObj = Val.getArrayInitializedElt(I); if (!visit(SubObj, CAT->getElementType(), Offset + I * ElemWidth)) return false; } // Next, initialize the rest of the array using the filler. if (Val.hasArrayFiller()) { const APValue &Filler = Val.getArrayFiller(); for (unsigned I = NumInitializedElts; I != ArraySize; ++I) { if (!visit(Filler, CAT->getElementType(), Offset + I * ElemWidth)) return false; } } return true; } bool visitInt(const APSInt &Val, QualType Ty, CharUnits Offset) { CharUnits Width = Info.Ctx.getTypeSizeInChars(Ty); SmallVector Bytes(Width.getQuantity()); llvm::StoreIntToMemory(Val, &*Bytes.begin(), Width.getQuantity()); Buffer.writeObject(Offset, Bytes); return true; } bool visitFloat(const APFloat &Val, QualType Ty, CharUnits Offset) { APSInt AsInt(Val.bitcastToAPInt()); return visitInt(AsInt, Ty, Offset); } public: static Optional convert(EvalInfo &Info, const APValue &Src, const CastExpr *BCE) { CharUnits DstSize = Info.Ctx.getTypeSizeInChars(BCE->getType()); APValueToBufferConverter Converter(Info, DstSize, BCE); if (!Converter.visit(Src, BCE->getSubExpr()->getType())) return None; return Converter.Buffer; } }; /// Write an BitCastBuffer into an APValue. class BufferToAPValueConverter { EvalInfo &Info; const BitCastBuffer &Buffer; const CastExpr *BCE; BufferToAPValueConverter(EvalInfo &Info, const BitCastBuffer &Buffer, const CastExpr *BCE) : Info(Info), Buffer(Buffer), BCE(BCE) {} // Emit an unsupported bit_cast type error. Sema refuses to build a bit_cast // with an invalid type, so anything left is a deficiency on our part (FIXME). // Ideally this will be unreachable. llvm::NoneType unsupportedType(QualType Ty) { Info.FFDiag(BCE->getBeginLoc(), diag::note_constexpr_bit_cast_unsupported_type) << Ty; return None; } Optional visit(const BuiltinType *T, CharUnits Offset, const EnumType *EnumSugar = nullptr) { if (T->isNullPtrType()) { uint64_t NullValue = Info.Ctx.getTargetNullPointerValue(QualType(T, 0)); return APValue((Expr *)nullptr, /*Offset=*/CharUnits::fromQuantity(NullValue), APValue::NoLValuePath{}, /*IsNullPtr=*/true); } CharUnits SizeOf = Info.Ctx.getTypeSizeInChars(T); SmallVector Bytes; if (!Buffer.readObject(Offset, SizeOf, Bytes)) { // If this is std::byte or unsigned char, then its okay to store an // indeterminate value. bool IsStdByte = EnumSugar && EnumSugar->isStdByteType(); bool IsUChar = !EnumSugar && (T->isSpecificBuiltinType(BuiltinType::UChar) || T->isSpecificBuiltinType(BuiltinType::Char_U)); if (!IsStdByte && !IsUChar) { QualType DisplayType(EnumSugar ? (const Type *)EnumSugar : T, 0); Info.FFDiag(BCE->getExprLoc(), diag::note_constexpr_bit_cast_indet_dest) << DisplayType << Info.Ctx.getLangOpts().CharIsSigned; return None; } return APValue::IndeterminateValue(); } APSInt Val(SizeOf.getQuantity() * Info.Ctx.getCharWidth(), true); llvm::LoadIntFromMemory(Val, &*Bytes.begin(), Bytes.size()); if (T->isIntegralOrEnumerationType()) { Val.setIsSigned(T->isSignedIntegerOrEnumerationType()); return APValue(Val); } if (T->isRealFloatingType()) { const llvm::fltSemantics &Semantics = Info.Ctx.getFloatTypeSemantics(QualType(T, 0)); return APValue(APFloat(Semantics, Val)); } return unsupportedType(QualType(T, 0)); } Optional visit(const RecordType *RTy, CharUnits Offset) { const RecordDecl *RD = RTy->getAsRecordDecl(); const ASTRecordLayout &Layout = Info.Ctx.getASTRecordLayout(RD); unsigned NumBases = 0; if (auto *CXXRD = dyn_cast(RD)) NumBases = CXXRD->getNumBases(); APValue ResultVal(APValue::UninitStruct(), NumBases, std::distance(RD->field_begin(), RD->field_end())); // Visit the base classes. if (auto *CXXRD = dyn_cast(RD)) { for (size_t I = 0, E = CXXRD->getNumBases(); I != E; ++I) { const CXXBaseSpecifier &BS = CXXRD->bases_begin()[I]; CXXRecordDecl *BaseDecl = BS.getType()->getAsCXXRecordDecl(); if (BaseDecl->isEmpty() || Info.Ctx.getASTRecordLayout(BaseDecl).getNonVirtualSize().isZero()) continue; Optional SubObj = visitType( BS.getType(), Layout.getBaseClassOffset(BaseDecl) + Offset); if (!SubObj) return None; ResultVal.getStructBase(I) = *SubObj; } } // Visit the fields. unsigned FieldIdx = 0; for (FieldDecl *FD : RD->fields()) { // FIXME: We don't currently support bit-fields. A lot of the logic for // this is in CodeGen, so we need to factor it around. if (FD->isBitField()) { Info.FFDiag(BCE->getBeginLoc(), diag::note_constexpr_bit_cast_unsupported_bitfield); return None; } uint64_t FieldOffsetBits = Layout.getFieldOffset(FieldIdx); assert(FieldOffsetBits % Info.Ctx.getCharWidth() == 0); CharUnits FieldOffset = CharUnits::fromQuantity(FieldOffsetBits / Info.Ctx.getCharWidth()) + Offset; QualType FieldTy = FD->getType(); Optional SubObj = visitType(FieldTy, FieldOffset); if (!SubObj) return None; ResultVal.getStructField(FieldIdx) = *SubObj; ++FieldIdx; } return ResultVal; } Optional visit(const EnumType *Ty, CharUnits Offset) { QualType RepresentationType = Ty->getDecl()->getIntegerType(); assert(!RepresentationType.isNull() && "enum forward decl should be caught by Sema"); const BuiltinType *AsBuiltin = RepresentationType.getCanonicalType()->getAs(); assert(AsBuiltin && "non-integral enum underlying type?"); // Recurse into the underlying type. Treat std::byte transparently as // unsigned char. return visit(AsBuiltin, Offset, /*EnumTy=*/Ty); } Optional visit(const ConstantArrayType *Ty, CharUnits Offset) { size_t Size = Ty->getSize().getLimitedValue(); CharUnits ElementWidth = Info.Ctx.getTypeSizeInChars(Ty->getElementType()); APValue ArrayValue(APValue::UninitArray(), Size, Size); for (size_t I = 0; I != Size; ++I) { Optional ElementValue = visitType(Ty->getElementType(), Offset + I * ElementWidth); if (!ElementValue) return None; ArrayValue.getArrayInitializedElt(I) = std::move(*ElementValue); } return ArrayValue; } Optional visit(const Type *Ty, CharUnits Offset) { return unsupportedType(QualType(Ty, 0)); } Optional visitType(QualType Ty, CharUnits Offset) { QualType Can = Ty.getCanonicalType(); switch (Can->getTypeClass()) { #define TYPE(Class, Base) \ case Type::Class: \ return visit(cast(Can.getTypePtr()), Offset); #define ABSTRACT_TYPE(Class, Base) #define NON_CANONICAL_TYPE(Class, Base) \ case Type::Class: \ llvm_unreachable("non-canonical type should be impossible!"); #define DEPENDENT_TYPE(Class, Base) \ case Type::Class: \ llvm_unreachable( \ "dependent types aren't supported in the constant evaluator!"); #define NON_CANONICAL_UNLESS_DEPENDENT(Class, Base) \ case Type::Class: \ llvm_unreachable("either dependent or not canonical!"); #include "clang/AST/TypeNodes.def" } llvm_unreachable("Unhandled Type::TypeClass"); } public: // Pull out a full value of type DstType. static Optional convert(EvalInfo &Info, BitCastBuffer &Buffer, const CastExpr *BCE) { BufferToAPValueConverter Converter(Info, Buffer, BCE); return Converter.visitType(BCE->getType(), CharUnits::fromQuantity(0)); } }; static bool checkBitCastConstexprEligibilityType(SourceLocation Loc, QualType Ty, EvalInfo *Info, const ASTContext &Ctx, bool CheckingDest) { Ty = Ty.getCanonicalType(); auto diag = [&](int Reason) { if (Info) Info->FFDiag(Loc, diag::note_constexpr_bit_cast_invalid_type) << CheckingDest << (Reason == 4) << Reason; return false; }; auto note = [&](int Construct, QualType NoteTy, SourceLocation NoteLoc) { if (Info) Info->Note(NoteLoc, diag::note_constexpr_bit_cast_invalid_subtype) << NoteTy << Construct << Ty; return false; }; if (Ty->isUnionType()) return diag(0); if (Ty->isPointerType()) return diag(1); if (Ty->isMemberPointerType()) return diag(2); if (Ty.isVolatileQualified()) return diag(3); if (RecordDecl *Record = Ty->getAsRecordDecl()) { if (auto *CXXRD = dyn_cast(Record)) { for (CXXBaseSpecifier &BS : CXXRD->bases()) if (!checkBitCastConstexprEligibilityType(Loc, BS.getType(), Info, Ctx, CheckingDest)) return note(1, BS.getType(), BS.getBeginLoc()); } for (FieldDecl *FD : Record->fields()) { if (FD->getType()->isReferenceType()) return diag(4); if (!checkBitCastConstexprEligibilityType(Loc, FD->getType(), Info, Ctx, CheckingDest)) return note(0, FD->getType(), FD->getBeginLoc()); } } if (Ty->isArrayType() && !checkBitCastConstexprEligibilityType(Loc, Ctx.getBaseElementType(Ty), Info, Ctx, CheckingDest)) return false; return true; } static bool checkBitCastConstexprEligibility(EvalInfo *Info, const ASTContext &Ctx, const CastExpr *BCE) { bool DestOK = checkBitCastConstexprEligibilityType( BCE->getBeginLoc(), BCE->getType(), Info, Ctx, true); bool SourceOK = DestOK && checkBitCastConstexprEligibilityType( BCE->getBeginLoc(), BCE->getSubExpr()->getType(), Info, Ctx, false); return SourceOK; } static bool handleLValueToRValueBitCast(EvalInfo &Info, APValue &DestValue, APValue &SourceValue, const CastExpr *BCE) { assert(CHAR_BIT == 8 && Info.Ctx.getTargetInfo().getCharWidth() == 8 && "no host or target supports non 8-bit chars"); assert(SourceValue.isLValue() && "LValueToRValueBitcast requires an lvalue operand!"); if (!checkBitCastConstexprEligibility(&Info, Info.Ctx, BCE)) return false; LValue SourceLValue; APValue SourceRValue; SourceLValue.setFrom(Info.Ctx, SourceValue); if (!handleLValueToRValueConversion(Info, BCE, BCE->getSubExpr()->getType().withConst(), SourceLValue, SourceRValue)) return false; // Read out SourceValue into a char buffer. Optional Buffer = APValueToBufferConverter::convert(Info, SourceRValue, BCE); if (!Buffer) return false; // Write out the buffer into a new APValue. Optional MaybeDestValue = BufferToAPValueConverter::convert(Info, *Buffer, BCE); if (!MaybeDestValue) return false; DestValue = std::move(*MaybeDestValue); return true; } template class ExprEvaluatorBase : public ConstStmtVisitor { private: Derived &getDerived() { return static_cast(*this); } bool DerivedSuccess(const APValue &V, const Expr *E) { return getDerived().Success(V, E); } bool DerivedZeroInitialization(const Expr *E) { return getDerived().ZeroInitialization(E); } // Check whether a conditional operator with a non-constant condition is a // potential constant expression. If neither arm is a potential constant // expression, then the conditional operator is not either. template void CheckPotentialConstantConditional(const ConditionalOperator *E) { assert(Info.checkingPotentialConstantExpression()); // Speculatively evaluate both arms. SmallVector Diag; { SpeculativeEvaluationRAII Speculate(Info, &Diag); StmtVisitorTy::Visit(E->getFalseExpr()); if (Diag.empty()) return; } { SpeculativeEvaluationRAII Speculate(Info, &Diag); Diag.clear(); StmtVisitorTy::Visit(E->getTrueExpr()); if (Diag.empty()) return; } Error(E, diag::note_constexpr_conditional_never_const); } template bool HandleConditionalOperator(const ConditionalOperator *E) { bool BoolResult; if (!EvaluateAsBooleanCondition(E->getCond(), BoolResult, Info)) { if (Info.checkingPotentialConstantExpression() && Info.noteFailure()) { CheckPotentialConstantConditional(E); return false; } if (Info.noteFailure()) { StmtVisitorTy::Visit(E->getTrueExpr()); StmtVisitorTy::Visit(E->getFalseExpr()); } return false; } Expr *EvalExpr = BoolResult ? E->getTrueExpr() : E->getFalseExpr(); return StmtVisitorTy::Visit(EvalExpr); } protected: EvalInfo &Info; typedef ConstStmtVisitor StmtVisitorTy; typedef ExprEvaluatorBase ExprEvaluatorBaseTy; OptionalDiagnostic CCEDiag(const Expr *E, diag::kind D) { return Info.CCEDiag(E, D); } bool ZeroInitialization(const Expr *E) { return Error(E); } public: ExprEvaluatorBase(EvalInfo &Info) : Info(Info) {} EvalInfo &getEvalInfo() { return Info; } /// Report an evaluation error. This should only be called when an error is /// first discovered. When propagating an error, just return false. bool Error(const Expr *E, diag::kind D) { Info.FFDiag(E, D); return false; } bool Error(const Expr *E) { return Error(E, diag::note_invalid_subexpr_in_const_expr); } bool VisitStmt(const Stmt *) { llvm_unreachable("Expression evaluator should not be called on stmts"); } bool VisitExpr(const Expr *E) { return Error(E); } bool VisitConstantExpr(const ConstantExpr *E) { return StmtVisitorTy::Visit(E->getSubExpr()); } bool VisitParenExpr(const ParenExpr *E) { return StmtVisitorTy::Visit(E->getSubExpr()); } bool VisitUnaryExtension(const UnaryOperator *E) { return StmtVisitorTy::Visit(E->getSubExpr()); } bool VisitUnaryPlus(const UnaryOperator *E) { return StmtVisitorTy::Visit(E->getSubExpr()); } bool VisitChooseExpr(const ChooseExpr *E) { return StmtVisitorTy::Visit(E->getChosenSubExpr()); } bool VisitGenericSelectionExpr(const GenericSelectionExpr *E) { return StmtVisitorTy::Visit(E->getResultExpr()); } bool VisitSubstNonTypeTemplateParmExpr(const SubstNonTypeTemplateParmExpr *E) { return StmtVisitorTy::Visit(E->getReplacement()); } bool VisitCXXDefaultArgExpr(const CXXDefaultArgExpr *E) { TempVersionRAII RAII(*Info.CurrentCall); SourceLocExprScopeGuard Guard(E, Info.CurrentCall->CurSourceLocExprScope); return StmtVisitorTy::Visit(E->getExpr()); } bool VisitCXXDefaultInitExpr(const CXXDefaultInitExpr *E) { TempVersionRAII RAII(*Info.CurrentCall); // The initializer may not have been parsed yet, or might be erroneous. if (!E->getExpr()) return Error(E); SourceLocExprScopeGuard Guard(E, Info.CurrentCall->CurSourceLocExprScope); return StmtVisitorTy::Visit(E->getExpr()); } // We cannot create any objects for which cleanups are required, so there is // nothing to do here; all cleanups must come from unevaluated subexpressions. bool VisitExprWithCleanups(const ExprWithCleanups *E) { return StmtVisitorTy::Visit(E->getSubExpr()); } bool VisitCXXReinterpretCastExpr(const CXXReinterpretCastExpr *E) { CCEDiag(E, diag::note_constexpr_invalid_cast) << 0; return static_cast(this)->VisitCastExpr(E); } bool VisitCXXDynamicCastExpr(const CXXDynamicCastExpr *E) { if (!Info.Ctx.getLangOpts().CPlusPlus2a) CCEDiag(E, diag::note_constexpr_invalid_cast) << 1; return static_cast(this)->VisitCastExpr(E); } bool VisitBuiltinBitCastExpr(const BuiltinBitCastExpr *E) { return static_cast(this)->VisitCastExpr(E); } bool VisitBinaryOperator(const BinaryOperator *E) { switch (E->getOpcode()) { default: return Error(E); case BO_Comma: VisitIgnoredValue(E->getLHS()); return StmtVisitorTy::Visit(E->getRHS()); case BO_PtrMemD: case BO_PtrMemI: { LValue Obj; if (!HandleMemberPointerAccess(Info, E, Obj)) return false; APValue Result; if (!handleLValueToRValueConversion(Info, E, E->getType(), Obj, Result)) return false; return DerivedSuccess(Result, E); } } } bool VisitBinaryConditionalOperator(const BinaryConditionalOperator *E) { // Evaluate and cache the common expression. We treat it as a temporary, // even though it's not quite the same thing. if (!Evaluate(Info.CurrentCall->createTemporary(E->getOpaqueValue(), false), Info, E->getCommon())) return false; return HandleConditionalOperator(E); } bool VisitConditionalOperator(const ConditionalOperator *E) { bool IsBcpCall = false; // If the condition (ignoring parens) is a __builtin_constant_p call, // the result is a constant expression if it can be folded without // side-effects. This is an important GNU extension. See GCC PR38377 // for discussion. if (const CallExpr *CallCE = dyn_cast(E->getCond()->IgnoreParenCasts())) if (CallCE->getBuiltinCallee() == Builtin::BI__builtin_constant_p) IsBcpCall = true; // Always assume __builtin_constant_p(...) ? ... : ... is a potential // constant expression; we can't check whether it's potentially foldable. // FIXME: We should instead treat __builtin_constant_p as non-constant if // it would return 'false' in this mode. if (Info.checkingPotentialConstantExpression() && IsBcpCall) return false; FoldConstant Fold(Info, IsBcpCall); if (!HandleConditionalOperator(E)) { Fold.keepDiagnostics(); return false; } return true; } bool VisitOpaqueValueExpr(const OpaqueValueExpr *E) { if (APValue *Value = Info.CurrentCall->getCurrentTemporary(E)) return DerivedSuccess(*Value, E); const Expr *Source = E->getSourceExpr(); if (!Source) return Error(E); if (Source == E) { // sanity checking. assert(0 && "OpaqueValueExpr recursively refers to itself"); return Error(E); } return StmtVisitorTy::Visit(Source); } bool VisitCallExpr(const CallExpr *E) { APValue Result; if (!handleCallExpr(E, Result, nullptr)) return false; return DerivedSuccess(Result, E); } bool handleCallExpr(const CallExpr *E, APValue &Result, const LValue *ResultSlot) { const Expr *Callee = E->getCallee()->IgnoreParens(); QualType CalleeType = Callee->getType(); const FunctionDecl *FD = nullptr; LValue *This = nullptr, ThisVal; auto Args = llvm::makeArrayRef(E->getArgs(), E->getNumArgs()); bool HasQualifier = false; // Extract function decl and 'this' pointer from the callee. if (CalleeType->isSpecificBuiltinType(BuiltinType::BoundMember)) { const CXXMethodDecl *Member = nullptr; if (const MemberExpr *ME = dyn_cast(Callee)) { // Explicit bound member calls, such as x.f() or p->g(); if (!EvaluateObjectArgument(Info, ME->getBase(), ThisVal)) return false; Member = dyn_cast(ME->getMemberDecl()); if (!Member) return Error(Callee); This = &ThisVal; HasQualifier = ME->hasQualifier(); } else if (const BinaryOperator *BE = dyn_cast(Callee)) { // Indirect bound member calls ('.*' or '->*'). Member = dyn_cast_or_null( HandleMemberPointerAccess(Info, BE, ThisVal, false)); if (!Member) return Error(Callee); This = &ThisVal; } else return Error(Callee); FD = Member; } else if (CalleeType->isFunctionPointerType()) { LValue Call; if (!EvaluatePointer(Callee, Call, Info)) return false; if (!Call.getLValueOffset().isZero()) return Error(Callee); FD = dyn_cast_or_null( Call.getLValueBase().dyn_cast()); if (!FD) return Error(Callee); // Don't call function pointers which have been cast to some other type. // Per DR (no number yet), the caller and callee can differ in noexcept. if (!Info.Ctx.hasSameFunctionTypeIgnoringExceptionSpec( CalleeType->getPointeeType(), FD->getType())) { return Error(E); } // Overloaded operator calls to member functions are represented as normal // calls with '*this' as the first argument. const CXXMethodDecl *MD = dyn_cast(FD); if (MD && !MD->isStatic()) { // FIXME: When selecting an implicit conversion for an overloaded // operator delete, we sometimes try to evaluate calls to conversion // operators without a 'this' parameter! if (Args.empty()) return Error(E); if (!EvaluateObjectArgument(Info, Args[0], ThisVal)) return false; This = &ThisVal; Args = Args.slice(1); } else if (MD && MD->isLambdaStaticInvoker()) { // Map the static invoker for the lambda back to the call operator. // Conveniently, we don't have to slice out the 'this' argument (as is // being done for the non-static case), since a static member function // doesn't have an implicit argument passed in. const CXXRecordDecl *ClosureClass = MD->getParent(); assert( ClosureClass->captures_begin() == ClosureClass->captures_end() && "Number of captures must be zero for conversion to function-ptr"); const CXXMethodDecl *LambdaCallOp = ClosureClass->getLambdaCallOperator(); // Set 'FD', the function that will be called below, to the call // operator. If the closure object represents a generic lambda, find // the corresponding specialization of the call operator. if (ClosureClass->isGenericLambda()) { assert(MD->isFunctionTemplateSpecialization() && "A generic lambda's static-invoker function must be a " "template specialization"); const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs(); FunctionTemplateDecl *CallOpTemplate = LambdaCallOp->getDescribedFunctionTemplate(); void *InsertPos = nullptr; FunctionDecl *CorrespondingCallOpSpecialization = CallOpTemplate->findSpecialization(TAL->asArray(), InsertPos); assert(CorrespondingCallOpSpecialization && "We must always have a function call operator specialization " "that corresponds to our static invoker specialization"); FD = cast(CorrespondingCallOpSpecialization); } else FD = LambdaCallOp; } } else return Error(E); SmallVector CovariantAdjustmentPath; if (This) { auto *NamedMember = dyn_cast(FD); if (NamedMember && NamedMember->isVirtual() && !HasQualifier) { // Perform virtual dispatch, if necessary. FD = HandleVirtualDispatch(Info, E, *This, NamedMember, CovariantAdjustmentPath); if (!FD) return false; } else { // Check that the 'this' pointer points to an object of the right type. if (!checkNonVirtualMemberCallThisPointer(Info, E, *This)) return false; } } const FunctionDecl *Definition = nullptr; Stmt *Body = FD->getBody(Definition); if (!CheckConstexprFunction(Info, E->getExprLoc(), FD, Definition, Body) || !HandleFunctionCall(E->getExprLoc(), Definition, This, Args, Body, Info, Result, ResultSlot)) return false; if (!CovariantAdjustmentPath.empty() && !HandleCovariantReturnAdjustment(Info, E, Result, CovariantAdjustmentPath)) return false; return true; } bool VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) { return StmtVisitorTy::Visit(E->getInitializer()); } bool VisitInitListExpr(const InitListExpr *E) { if (E->getNumInits() == 0) return DerivedZeroInitialization(E); if (E->getNumInits() == 1) return StmtVisitorTy::Visit(E->getInit(0)); return Error(E); } bool VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E) { return DerivedZeroInitialization(E); } bool VisitCXXScalarValueInitExpr(const CXXScalarValueInitExpr *E) { return DerivedZeroInitialization(E); } bool VisitCXXNullPtrLiteralExpr(const CXXNullPtrLiteralExpr *E) { return DerivedZeroInitialization(E); } /// A member expression where the object is a prvalue is itself a prvalue. bool VisitMemberExpr(const MemberExpr *E) { assert(!Info.Ctx.getLangOpts().CPlusPlus11 && "missing temporary materialization conversion"); assert(!E->isArrow() && "missing call to bound member function?"); APValue Val; if (!Evaluate(Val, Info, E->getBase())) return false; QualType BaseTy = E->getBase()->getType(); const FieldDecl *FD = dyn_cast(E->getMemberDecl()); if (!FD) return Error(E); assert(!FD->getType()->isReferenceType() && "prvalue reference?"); assert(BaseTy->castAs()->getDecl()->getCanonicalDecl() == FD->getParent()->getCanonicalDecl() && "record / field mismatch"); // Note: there is no lvalue base here. But this case should only ever // happen in C or in C++98, where we cannot be evaluating a constexpr // constructor, which is the only case the base matters. CompleteObject Obj(APValue::LValueBase(), &Val, BaseTy); SubobjectDesignator Designator(BaseTy); Designator.addDeclUnchecked(FD); APValue Result; return extractSubobject(Info, E, Obj, Designator, Result) && DerivedSuccess(Result, E); } bool VisitCastExpr(const CastExpr *E) { switch (E->getCastKind()) { default: break; case CK_AtomicToNonAtomic: { APValue AtomicVal; // This does not need to be done in place even for class/array types: // atomic-to-non-atomic conversion implies copying the object // representation. if (!Evaluate(AtomicVal, Info, E->getSubExpr())) return false; return DerivedSuccess(AtomicVal, E); } case CK_NoOp: case CK_UserDefinedConversion: return StmtVisitorTy::Visit(E->getSubExpr()); case CK_LValueToRValue: { LValue LVal; if (!EvaluateLValue(E->getSubExpr(), LVal, Info)) return false; APValue RVal; // Note, we use the subexpression's type in order to retain cv-qualifiers. if (!handleLValueToRValueConversion(Info, E, E->getSubExpr()->getType(), LVal, RVal)) return false; return DerivedSuccess(RVal, E); } case CK_LValueToRValueBitCast: { APValue DestValue, SourceValue; if (!Evaluate(SourceValue, Info, E->getSubExpr())) return false; if (!handleLValueToRValueBitCast(Info, DestValue, SourceValue, E)) return false; return DerivedSuccess(DestValue, E); } } return Error(E); } bool VisitUnaryPostInc(const UnaryOperator *UO) { return VisitUnaryPostIncDec(UO); } bool VisitUnaryPostDec(const UnaryOperator *UO) { return VisitUnaryPostIncDec(UO); } bool VisitUnaryPostIncDec(const UnaryOperator *UO) { if (!Info.getLangOpts().CPlusPlus14 && !Info.keepEvaluatingAfterFailure()) return Error(UO); LValue LVal; if (!EvaluateLValue(UO->getSubExpr(), LVal, Info)) return false; APValue RVal; if (!handleIncDec(this->Info, UO, LVal, UO->getSubExpr()->getType(), UO->isIncrementOp(), &RVal)) return false; return DerivedSuccess(RVal, UO); } bool VisitStmtExpr(const StmtExpr *E) { // We will have checked the full-expressions inside the statement expression // when they were completed, and don't need to check them again now. if (Info.checkingForUndefinedBehavior()) return Error(E); BlockScopeRAII Scope(Info); const CompoundStmt *CS = E->getSubStmt(); if (CS->body_empty()) return true; for (CompoundStmt::const_body_iterator BI = CS->body_begin(), BE = CS->body_end(); /**/; ++BI) { if (BI + 1 == BE) { const Expr *FinalExpr = dyn_cast(*BI); if (!FinalExpr) { Info.FFDiag((*BI)->getBeginLoc(), diag::note_constexpr_stmt_expr_unsupported); return false; } return this->Visit(FinalExpr); } APValue ReturnValue; StmtResult Result = { ReturnValue, nullptr }; EvalStmtResult ESR = EvaluateStmt(Result, Info, *BI); if (ESR != ESR_Succeeded) { // FIXME: If the statement-expression terminated due to 'return', // 'break', or 'continue', it would be nice to propagate that to // the outer statement evaluation rather than bailing out. if (ESR != ESR_Failed) Info.FFDiag((*BI)->getBeginLoc(), diag::note_constexpr_stmt_expr_unsupported); return false; } } llvm_unreachable("Return from function from the loop above."); } /// Visit a value which is evaluated, but whose value is ignored. void VisitIgnoredValue(const Expr *E) { EvaluateIgnoredValue(Info, E); } /// Potentially visit a MemberExpr's base expression. void VisitIgnoredBaseExpression(const Expr *E) { // While MSVC doesn't evaluate the base expression, it does diagnose the // presence of side-effecting behavior. if (Info.getLangOpts().MSVCCompat && !E->HasSideEffects(Info.Ctx)) return; VisitIgnoredValue(E); } }; } // namespace //===----------------------------------------------------------------------===// // Common base class for lvalue and temporary evaluation. //===----------------------------------------------------------------------===// namespace { template class LValueExprEvaluatorBase : public ExprEvaluatorBase { protected: LValue &Result; bool InvalidBaseOK; typedef LValueExprEvaluatorBase LValueExprEvaluatorBaseTy; typedef ExprEvaluatorBase ExprEvaluatorBaseTy; bool Success(APValue::LValueBase B) { Result.set(B); return true; } bool evaluatePointer(const Expr *E, LValue &Result) { return EvaluatePointer(E, Result, this->Info, InvalidBaseOK); } public: LValueExprEvaluatorBase(EvalInfo &Info, LValue &Result, bool InvalidBaseOK) : ExprEvaluatorBaseTy(Info), Result(Result), InvalidBaseOK(InvalidBaseOK) {} bool Success(const APValue &V, const Expr *E) { Result.setFrom(this->Info.Ctx, V); return true; } bool VisitMemberExpr(const MemberExpr *E) { // Handle non-static data members. QualType BaseTy; bool EvalOK; if (E->isArrow()) { EvalOK = evaluatePointer(E->getBase(), Result); BaseTy = E->getBase()->getType()->castAs()->getPointeeType(); } else if (E->getBase()->isRValue()) { assert(E->getBase()->getType()->isRecordType()); EvalOK = EvaluateTemporary(E->getBase(), Result, this->Info); BaseTy = E->getBase()->getType(); } else { EvalOK = this->Visit(E->getBase()); BaseTy = E->getBase()->getType(); } if (!EvalOK) { if (!InvalidBaseOK) return false; Result.setInvalid(E); return true; } const ValueDecl *MD = E->getMemberDecl(); if (const FieldDecl *FD = dyn_cast(E->getMemberDecl())) { assert(BaseTy->getAs()->getDecl()->getCanonicalDecl() == FD->getParent()->getCanonicalDecl() && "record / field mismatch"); (void)BaseTy; if (!HandleLValueMember(this->Info, E, Result, FD)) return false; } else if (const IndirectFieldDecl *IFD = dyn_cast(MD)) { if (!HandleLValueIndirectMember(this->Info, E, Result, IFD)) return false; } else return this->Error(E); if (MD->getType()->isReferenceType()) { APValue RefValue; if (!handleLValueToRValueConversion(this->Info, E, MD->getType(), Result, RefValue)) return false; return Success(RefValue, E); } return true; } bool VisitBinaryOperator(const BinaryOperator *E) { switch (E->getOpcode()) { default: return ExprEvaluatorBaseTy::VisitBinaryOperator(E); case BO_PtrMemD: case BO_PtrMemI: return HandleMemberPointerAccess(this->Info, E, Result); } } bool VisitCastExpr(const CastExpr *E) { switch (E->getCastKind()) { default: return ExprEvaluatorBaseTy::VisitCastExpr(E); case CK_DerivedToBase: case CK_UncheckedDerivedToBase: if (!this->Visit(E->getSubExpr())) return false; // Now figure out the necessary offset to add to the base LV to get from // the derived class to the base class. return HandleLValueBasePath(this->Info, E, E->getSubExpr()->getType(), Result); } } }; } //===----------------------------------------------------------------------===// // LValue Evaluation // // This is used for evaluating lvalues (in C and C++), xvalues (in C++11), // function designators (in C), decl references to void objects (in C), and // temporaries (if building with -Wno-address-of-temporary). // // LValue evaluation produces values comprising a base expression of one of the // following types: // - Declarations // * VarDecl // * FunctionDecl // - Literals // * CompoundLiteralExpr in C (and in global scope in C++) // * StringLiteral // * PredefinedExpr // * ObjCStringLiteralExpr // * ObjCEncodeExpr // * AddrLabelExpr // * BlockExpr // * CallExpr for a MakeStringConstant builtin // - typeid(T) expressions, as TypeInfoLValues // - Locals and temporaries // * MaterializeTemporaryExpr // * Any Expr, with a CallIndex indicating the function in which the temporary // was evaluated, for cases where the MaterializeTemporaryExpr is missing // from the AST (FIXME). // * A MaterializeTemporaryExpr that has static storage duration, with no // CallIndex, for a lifetime-extended temporary. // plus an offset in bytes. //===----------------------------------------------------------------------===// namespace { class LValueExprEvaluator : public LValueExprEvaluatorBase { public: LValueExprEvaluator(EvalInfo &Info, LValue &Result, bool InvalidBaseOK) : LValueExprEvaluatorBaseTy(Info, Result, InvalidBaseOK) {} bool VisitVarDecl(const Expr *E, const VarDecl *VD); bool VisitUnaryPreIncDec(const UnaryOperator *UO); bool VisitDeclRefExpr(const DeclRefExpr *E); bool VisitPredefinedExpr(const PredefinedExpr *E) { return Success(E); } bool VisitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E); bool VisitCompoundLiteralExpr(const CompoundLiteralExpr *E); bool VisitMemberExpr(const MemberExpr *E); bool VisitStringLiteral(const StringLiteral *E) { return Success(E); } bool VisitObjCEncodeExpr(const ObjCEncodeExpr *E) { return Success(E); } bool VisitCXXTypeidExpr(const CXXTypeidExpr *E); bool VisitCXXUuidofExpr(const CXXUuidofExpr *E); bool VisitArraySubscriptExpr(const ArraySubscriptExpr *E); bool VisitUnaryDeref(const UnaryOperator *E); bool VisitUnaryReal(const UnaryOperator *E); bool VisitUnaryImag(const UnaryOperator *E); bool VisitUnaryPreInc(const UnaryOperator *UO) { return VisitUnaryPreIncDec(UO); } bool VisitUnaryPreDec(const UnaryOperator *UO) { return VisitUnaryPreIncDec(UO); } bool VisitBinAssign(const BinaryOperator *BO); bool VisitCompoundAssignOperator(const CompoundAssignOperator *CAO); bool VisitCastExpr(const CastExpr *E) { switch (E->getCastKind()) { default: return LValueExprEvaluatorBaseTy::VisitCastExpr(E); case CK_LValueBitCast: this->CCEDiag(E, diag::note_constexpr_invalid_cast) << 2; if (!Visit(E->getSubExpr())) return false; Result.Designator.setInvalid(); return true; case CK_BaseToDerived: if (!Visit(E->getSubExpr())) return false; return HandleBaseToDerivedCast(Info, E, Result); case CK_Dynamic: if (!Visit(E->getSubExpr())) return false; return HandleDynamicCast(Info, cast(E), Result); } } }; } // end anonymous namespace /// Evaluate an expression as an lvalue. This can be legitimately called on /// expressions which are not glvalues, in three cases: /// * function designators in C, and /// * "extern void" objects /// * @selector() expressions in Objective-C static bool EvaluateLValue(const Expr *E, LValue &Result, EvalInfo &Info, bool InvalidBaseOK) { assert(E->isGLValue() || E->getType()->isFunctionType() || E->getType()->isVoidType() || isa(E)); return LValueExprEvaluator(Info, Result, InvalidBaseOK).Visit(E); } bool LValueExprEvaluator::VisitDeclRefExpr(const DeclRefExpr *E) { if (const FunctionDecl *FD = dyn_cast(E->getDecl())) return Success(FD); if (const VarDecl *VD = dyn_cast(E->getDecl())) return VisitVarDecl(E, VD); if (const BindingDecl *BD = dyn_cast(E->getDecl())) return Visit(BD->getBinding()); return Error(E); } bool LValueExprEvaluator::VisitVarDecl(const Expr *E, const VarDecl *VD) { // If we are within a lambda's call operator, check whether the 'VD' referred // to within 'E' actually represents a lambda-capture that maps to a // data-member/field within the closure object, and if so, evaluate to the // field or what the field refers to. if (Info.CurrentCall && isLambdaCallOperator(Info.CurrentCall->Callee) && isa(E) && cast(E)->refersToEnclosingVariableOrCapture()) { // We don't always have a complete capture-map when checking or inferring if // the function call operator meets the requirements of a constexpr function // - but we don't need to evaluate the captures to determine constexprness // (dcl.constexpr C++17). if (Info.checkingPotentialConstantExpression()) return false; if (auto *FD = Info.CurrentCall->LambdaCaptureFields.lookup(VD)) { // Start with 'Result' referring to the complete closure object... Result = *Info.CurrentCall->This; // ... then update it to refer to the field of the closure object // that represents the capture. if (!HandleLValueMember(Info, E, Result, FD)) return false; // And if the field is of reference type, update 'Result' to refer to what // the field refers to. if (FD->getType()->isReferenceType()) { APValue RVal; if (!handleLValueToRValueConversion(Info, E, FD->getType(), Result, RVal)) return false; Result.setFrom(Info.Ctx, RVal); } return true; } } CallStackFrame *Frame = nullptr; if (VD->hasLocalStorage() && Info.CurrentCall->Index > 1) { // Only if a local variable was declared in the function currently being // evaluated, do we expect to be able to find its value in the current // frame. (Otherwise it was likely declared in an enclosing context and // could either have a valid evaluatable value (for e.g. a constexpr // variable) or be ill-formed (and trigger an appropriate evaluation // diagnostic)). if (Info.CurrentCall->Callee && Info.CurrentCall->Callee->Equals(VD->getDeclContext())) { Frame = Info.CurrentCall; } } if (!VD->getType()->isReferenceType()) { if (Frame) { Result.set({VD, Frame->Index, Info.CurrentCall->getCurrentTemporaryVersion(VD)}); return true; } return Success(VD); } APValue *V; if (!evaluateVarDeclInit(Info, E, VD, Frame, V, nullptr)) return false; if (!V->hasValue()) { // FIXME: Is it possible for V to be indeterminate here? If so, we should // adjust the diagnostic to say that. if (!Info.checkingPotentialConstantExpression()) Info.FFDiag(E, diag::note_constexpr_use_uninit_reference); return false; } return Success(*V, E); } bool LValueExprEvaluator::VisitMaterializeTemporaryExpr( const MaterializeTemporaryExpr *E) { // Walk through the expression to find the materialized temporary itself. SmallVector CommaLHSs; SmallVector Adjustments; const Expr *Inner = E->GetTemporaryExpr()-> skipRValueSubobjectAdjustments(CommaLHSs, Adjustments); // If we passed any comma operators, evaluate their LHSs. for (unsigned I = 0, N = CommaLHSs.size(); I != N; ++I) if (!EvaluateIgnoredValue(Info, CommaLHSs[I])) return false; // A materialized temporary with static storage duration can appear within the // result of a constant expression evaluation, so we need to preserve its // value for use outside this evaluation. APValue *Value; if (E->getStorageDuration() == SD_Static) { Value = Info.Ctx.getMaterializedTemporaryValue(E, true); *Value = APValue(); Result.set(E); } else { Value = &createTemporary(E, E->getStorageDuration() == SD_Automatic, Result, *Info.CurrentCall); } QualType Type = Inner->getType(); // Materialize the temporary itself. if (!EvaluateInPlace(*Value, Info, Result, Inner) || (E->getStorageDuration() == SD_Static && !CheckConstantExpression(Info, E->getExprLoc(), Type, *Value))) { *Value = APValue(); return false; } // Adjust our lvalue to refer to the desired subobject. for (unsigned I = Adjustments.size(); I != 0; /**/) { --I; switch (Adjustments[I].Kind) { case SubobjectAdjustment::DerivedToBaseAdjustment: if (!HandleLValueBasePath(Info, Adjustments[I].DerivedToBase.BasePath, Type, Result)) return false; Type = Adjustments[I].DerivedToBase.BasePath->getType(); break; case SubobjectAdjustment::FieldAdjustment: if (!HandleLValueMember(Info, E, Result, Adjustments[I].Field)) return false; Type = Adjustments[I].Field->getType(); break; case SubobjectAdjustment::MemberPointerAdjustment: if (!HandleMemberPointerAccess(this->Info, Type, Result, Adjustments[I].Ptr.RHS)) return false; Type = Adjustments[I].Ptr.MPT->getPointeeType(); break; } } return true; } bool LValueExprEvaluator::VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) { assert((!Info.getLangOpts().CPlusPlus || E->isFileScope()) && "lvalue compound literal in c++?"); // Defer visiting the literal until the lvalue-to-rvalue conversion. We can // only see this when folding in C, so there's no standard to follow here. return Success(E); } bool LValueExprEvaluator::VisitCXXTypeidExpr(const CXXTypeidExpr *E) { TypeInfoLValue TypeInfo; if (!E->isPotentiallyEvaluated()) { if (E->isTypeOperand()) TypeInfo = TypeInfoLValue(E->getTypeOperand(Info.Ctx).getTypePtr()); else TypeInfo = TypeInfoLValue(E->getExprOperand()->getType().getTypePtr()); } else { if (!Info.Ctx.getLangOpts().CPlusPlus2a) { Info.CCEDiag(E, diag::note_constexpr_typeid_polymorphic) << E->getExprOperand()->getType() << E->getExprOperand()->getSourceRange(); } if (!Visit(E->getExprOperand())) return false; Optional DynType = ComputeDynamicType(Info, E, Result, AK_TypeId); if (!DynType) return false; TypeInfo = TypeInfoLValue(Info.Ctx.getRecordType(DynType->Type).getTypePtr()); } return Success(APValue::LValueBase::getTypeInfo(TypeInfo, E->getType())); } bool LValueExprEvaluator::VisitCXXUuidofExpr(const CXXUuidofExpr *E) { return Success(E); } bool LValueExprEvaluator::VisitMemberExpr(const MemberExpr *E) { // Handle static data members. if (const VarDecl *VD = dyn_cast(E->getMemberDecl())) { VisitIgnoredBaseExpression(E->getBase()); return VisitVarDecl(E, VD); } // Handle static member functions. if (const CXXMethodDecl *MD = dyn_cast(E->getMemberDecl())) { if (MD->isStatic()) { VisitIgnoredBaseExpression(E->getBase()); return Success(MD); } } // Handle non-static data members. return LValueExprEvaluatorBaseTy::VisitMemberExpr(E); } bool LValueExprEvaluator::VisitArraySubscriptExpr(const ArraySubscriptExpr *E) { // FIXME: Deal with vectors as array subscript bases. if (E->getBase()->getType()->isVectorType()) return Error(E); bool Success = true; if (!evaluatePointer(E->getBase(), Result)) { if (!Info.noteFailure()) return false; Success = false; } APSInt Index; if (!EvaluateInteger(E->getIdx(), Index, Info)) return false; return Success && HandleLValueArrayAdjustment(Info, E, Result, E->getType(), Index); } bool LValueExprEvaluator::VisitUnaryDeref(const UnaryOperator *E) { return evaluatePointer(E->getSubExpr(), Result); } bool LValueExprEvaluator::VisitUnaryReal(const UnaryOperator *E) { if (!Visit(E->getSubExpr())) return false; // __real is a no-op on scalar lvalues. if (E->getSubExpr()->getType()->isAnyComplexType()) HandleLValueComplexElement(Info, E, Result, E->getType(), false); return true; } bool LValueExprEvaluator::VisitUnaryImag(const UnaryOperator *E) { assert(E->getSubExpr()->getType()->isAnyComplexType() && "lvalue __imag__ on scalar?"); if (!Visit(E->getSubExpr())) return false; HandleLValueComplexElement(Info, E, Result, E->getType(), true); return true; } bool LValueExprEvaluator::VisitUnaryPreIncDec(const UnaryOperator *UO) { if (!Info.getLangOpts().CPlusPlus14 && !Info.keepEvaluatingAfterFailure()) return Error(UO); if (!this->Visit(UO->getSubExpr())) return false; return handleIncDec( this->Info, UO, Result, UO->getSubExpr()->getType(), UO->isIncrementOp(), nullptr); } bool LValueExprEvaluator::VisitCompoundAssignOperator( const CompoundAssignOperator *CAO) { if (!Info.getLangOpts().CPlusPlus14 && !Info.keepEvaluatingAfterFailure()) return Error(CAO); APValue RHS; // The overall lvalue result is the result of evaluating the LHS. if (!this->Visit(CAO->getLHS())) { if (Info.noteFailure()) Evaluate(RHS, this->Info, CAO->getRHS()); return false; } if (!Evaluate(RHS, this->Info, CAO->getRHS())) return false; return handleCompoundAssignment( this->Info, CAO, Result, CAO->getLHS()->getType(), CAO->getComputationLHSType(), CAO->getOpForCompoundAssignment(CAO->getOpcode()), RHS); } bool LValueExprEvaluator::VisitBinAssign(const BinaryOperator *E) { if (!Info.getLangOpts().CPlusPlus14 && !Info.keepEvaluatingAfterFailure()) return Error(E); APValue NewVal; if (!this->Visit(E->getLHS())) { if (Info.noteFailure()) Evaluate(NewVal, this->Info, E->getRHS()); return false; } if (!Evaluate(NewVal, this->Info, E->getRHS())) return false; if (Info.getLangOpts().CPlusPlus2a && !HandleUnionActiveMemberChange(Info, E->getLHS(), Result)) return false; return handleAssignment(this->Info, E, Result, E->getLHS()->getType(), NewVal); } //===----------------------------------------------------------------------===// // Pointer Evaluation //===----------------------------------------------------------------------===// /// Attempts to compute the number of bytes available at the pointer /// returned by a function with the alloc_size attribute. Returns true if we /// were successful. Places an unsigned number into `Result`. /// /// This expects the given CallExpr to be a call to a function with an /// alloc_size attribute. static bool getBytesReturnedByAllocSizeCall(const ASTContext &Ctx, const CallExpr *Call, llvm::APInt &Result) { const AllocSizeAttr *AllocSize = getAllocSizeAttr(Call); assert(AllocSize && AllocSize->getElemSizeParam().isValid()); unsigned SizeArgNo = AllocSize->getElemSizeParam().getASTIndex(); unsigned BitsInSizeT = Ctx.getTypeSize(Ctx.getSizeType()); if (Call->getNumArgs() <= SizeArgNo) return false; auto EvaluateAsSizeT = [&](const Expr *E, APSInt &Into) { Expr::EvalResult ExprResult; if (!E->EvaluateAsInt(ExprResult, Ctx, Expr::SE_AllowSideEffects)) return false; Into = ExprResult.Val.getInt(); if (Into.isNegative() || !Into.isIntN(BitsInSizeT)) return false; Into = Into.zextOrSelf(BitsInSizeT); return true; }; APSInt SizeOfElem; if (!EvaluateAsSizeT(Call->getArg(SizeArgNo), SizeOfElem)) return false; if (!AllocSize->getNumElemsParam().isValid()) { Result = std::move(SizeOfElem); return true; } APSInt NumberOfElems; unsigned NumArgNo = AllocSize->getNumElemsParam().getASTIndex(); if (!EvaluateAsSizeT(Call->getArg(NumArgNo), NumberOfElems)) return false; bool Overflow; llvm::APInt BytesAvailable = SizeOfElem.umul_ov(NumberOfElems, Overflow); if (Overflow) return false; Result = std::move(BytesAvailable); return true; } /// Convenience function. LVal's base must be a call to an alloc_size /// function. static bool getBytesReturnedByAllocSizeCall(const ASTContext &Ctx, const LValue &LVal, llvm::APInt &Result) { assert(isBaseAnAllocSizeCall(LVal.getLValueBase()) && "Can't get the size of a non alloc_size function"); const auto *Base = LVal.getLValueBase().get(); const CallExpr *CE = tryUnwrapAllocSizeCall(Base); return getBytesReturnedByAllocSizeCall(Ctx, CE, Result); } /// Attempts to evaluate the given LValueBase as the result of a call to /// a function with the alloc_size attribute. If it was possible to do so, this /// function will return true, make Result's Base point to said function call, /// and mark Result's Base as invalid. static bool evaluateLValueAsAllocSize(EvalInfo &Info, APValue::LValueBase Base, LValue &Result) { if (Base.isNull()) return false; // Because we do no form of static analysis, we only support const variables. // // Additionally, we can't support parameters, nor can we support static // variables (in the latter case, use-before-assign isn't UB; in the former, // we have no clue what they'll be assigned to). const auto *VD = dyn_cast_or_null(Base.dyn_cast()); if (!VD || !VD->isLocalVarDecl() || !VD->getType().isConstQualified()) return false; const Expr *Init = VD->getAnyInitializer(); if (!Init) return false; const Expr *E = Init->IgnoreParens(); if (!tryUnwrapAllocSizeCall(E)) return false; // Store E instead of E unwrapped so that the type of the LValue's base is // what the user wanted. Result.setInvalid(E); QualType Pointee = E->getType()->castAs()->getPointeeType(); Result.addUnsizedArray(Info, E, Pointee); return true; } namespace { class PointerExprEvaluator : public ExprEvaluatorBase { LValue &Result; bool InvalidBaseOK; bool Success(const Expr *E) { Result.set(E); return true; } bool evaluateLValue(const Expr *E, LValue &Result) { return EvaluateLValue(E, Result, Info, InvalidBaseOK); } bool evaluatePointer(const Expr *E, LValue &Result) { return EvaluatePointer(E, Result, Info, InvalidBaseOK); } bool visitNonBuiltinCallExpr(const CallExpr *E); public: PointerExprEvaluator(EvalInfo &info, LValue &Result, bool InvalidBaseOK) : ExprEvaluatorBaseTy(info), Result(Result), InvalidBaseOK(InvalidBaseOK) {} bool Success(const APValue &V, const Expr *E) { Result.setFrom(Info.Ctx, V); return true; } bool ZeroInitialization(const Expr *E) { auto TargetVal = Info.Ctx.getTargetNullPointerValue(E->getType()); Result.setNull(E->getType(), TargetVal); return true; } bool VisitBinaryOperator(const BinaryOperator *E); bool VisitCastExpr(const CastExpr* E); bool VisitUnaryAddrOf(const UnaryOperator *E); bool VisitObjCStringLiteral(const ObjCStringLiteral *E) { return Success(E); } bool VisitObjCBoxedExpr(const ObjCBoxedExpr *E) { if (E->isExpressibleAsConstantInitializer()) return Success(E); if (Info.noteFailure()) EvaluateIgnoredValue(Info, E->getSubExpr()); return Error(E); } bool VisitAddrLabelExpr(const AddrLabelExpr *E) { return Success(E); } bool VisitCallExpr(const CallExpr *E); bool VisitBuiltinCallExpr(const CallExpr *E, unsigned BuiltinOp); bool VisitBlockExpr(const BlockExpr *E) { if (!E->getBlockDecl()->hasCaptures()) return Success(E); return Error(E); } bool VisitCXXThisExpr(const CXXThisExpr *E) { // Can't look at 'this' when checking a potential constant expression. if (Info.checkingPotentialConstantExpression()) return false; if (!Info.CurrentCall->This) { if (Info.getLangOpts().CPlusPlus11) Info.FFDiag(E, diag::note_constexpr_this) << E->isImplicit(); else Info.FFDiag(E); return false; } Result = *Info.CurrentCall->This; // If we are inside a lambda's call operator, the 'this' expression refers // to the enclosing '*this' object (either by value or reference) which is // either copied into the closure object's field that represents the '*this' // or refers to '*this'. if (isLambdaCallOperator(Info.CurrentCall->Callee)) { // Update 'Result' to refer to the data member/field of the closure object // that represents the '*this' capture. if (!HandleLValueMember(Info, E, Result, Info.CurrentCall->LambdaThisCaptureField)) return false; // If we captured '*this' by reference, replace the field with its referent. if (Info.CurrentCall->LambdaThisCaptureField->getType() ->isPointerType()) { APValue RVal; if (!handleLValueToRValueConversion(Info, E, E->getType(), Result, RVal)) return false; Result.setFrom(Info.Ctx, RVal); } } return true; } bool VisitSourceLocExpr(const SourceLocExpr *E) { assert(E->isStringType() && "SourceLocExpr isn't a pointer type?"); APValue LValResult = E->EvaluateInContext( Info.Ctx, Info.CurrentCall->CurSourceLocExprScope.getDefaultExpr()); Result.setFrom(Info.Ctx, LValResult); return true; } // FIXME: Missing: @protocol, @selector }; } // end anonymous namespace static bool EvaluatePointer(const Expr* E, LValue& Result, EvalInfo &Info, bool InvalidBaseOK) { assert(E->isRValue() && E->getType()->hasPointerRepresentation()); return PointerExprEvaluator(Info, Result, InvalidBaseOK).Visit(E); } bool PointerExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) { if (E->getOpcode() != BO_Add && E->getOpcode() != BO_Sub) return ExprEvaluatorBaseTy::VisitBinaryOperator(E); const Expr *PExp = E->getLHS(); const Expr *IExp = E->getRHS(); if (IExp->getType()->isPointerType()) std::swap(PExp, IExp); bool EvalPtrOK = evaluatePointer(PExp, Result); if (!EvalPtrOK && !Info.noteFailure()) return false; llvm::APSInt Offset; if (!EvaluateInteger(IExp, Offset, Info) || !EvalPtrOK) return false; if (E->getOpcode() == BO_Sub) negateAsSigned(Offset); QualType Pointee = PExp->getType()->castAs()->getPointeeType(); return HandleLValueArrayAdjustment(Info, E, Result, Pointee, Offset); } bool PointerExprEvaluator::VisitUnaryAddrOf(const UnaryOperator *E) { return evaluateLValue(E->getSubExpr(), Result); } bool PointerExprEvaluator::VisitCastExpr(const CastExpr *E) { const Expr *SubExpr = E->getSubExpr(); switch (E->getCastKind()) { default: break; case CK_BitCast: case CK_CPointerToObjCPointerCast: case CK_BlockPointerToObjCPointerCast: case CK_AnyPointerToBlockPointerCast: case CK_AddressSpaceConversion: if (!Visit(SubExpr)) return false; // Bitcasts to cv void* are static_casts, not reinterpret_casts, so are // permitted in constant expressions in C++11. Bitcasts from cv void* are // also static_casts, but we disallow them as a resolution to DR1312. if (!E->getType()->isVoidPointerType()) { Result.Designator.setInvalid(); if (SubExpr->getType()->isVoidPointerType()) CCEDiag(E, diag::note_constexpr_invalid_cast) << 3 << SubExpr->getType(); else CCEDiag(E, diag::note_constexpr_invalid_cast) << 2; } if (E->getCastKind() == CK_AddressSpaceConversion && Result.IsNullPtr) ZeroInitialization(E); return true; case CK_DerivedToBase: case CK_UncheckedDerivedToBase: if (!evaluatePointer(E->getSubExpr(), Result)) return false; if (!Result.Base && Result.Offset.isZero()) return true; // Now figure out the necessary offset to add to the base LV to get from // the derived class to the base class. return HandleLValueBasePath(Info, E, E->getSubExpr()->getType()-> castAs()->getPointeeType(), Result); case CK_BaseToDerived: if (!Visit(E->getSubExpr())) return false; if (!Result.Base && Result.Offset.isZero()) return true; return HandleBaseToDerivedCast(Info, E, Result); case CK_Dynamic: if (!Visit(E->getSubExpr())) return false; return HandleDynamicCast(Info, cast(E), Result); case CK_NullToPointer: VisitIgnoredValue(E->getSubExpr()); return ZeroInitialization(E); case CK_IntegralToPointer: { CCEDiag(E, diag::note_constexpr_invalid_cast) << 2; APValue Value; if (!EvaluateIntegerOrLValue(SubExpr, Value, Info)) break; if (Value.isInt()) { unsigned Size = Info.Ctx.getTypeSize(E->getType()); uint64_t N = Value.getInt().extOrTrunc(Size).getZExtValue(); Result.Base = (Expr*)nullptr; Result.InvalidBase = false; Result.Offset = CharUnits::fromQuantity(N); Result.Designator.setInvalid(); Result.IsNullPtr = false; return true; } else { // Cast is of an lvalue, no need to change value. Result.setFrom(Info.Ctx, Value); return true; } } case CK_ArrayToPointerDecay: { if (SubExpr->isGLValue()) { if (!evaluateLValue(SubExpr, Result)) return false; } else { APValue &Value = createTemporary(SubExpr, false, Result, *Info.CurrentCall); if (!EvaluateInPlace(Value, Info, Result, SubExpr)) return false; } // The result is a pointer to the first element of the array. auto *AT = Info.Ctx.getAsArrayType(SubExpr->getType()); if (auto *CAT = dyn_cast(AT)) Result.addArray(Info, E, CAT); else Result.addUnsizedArray(Info, E, AT->getElementType()); return true; } case CK_FunctionToPointerDecay: return evaluateLValue(SubExpr, Result); case CK_LValueToRValue: { LValue LVal; if (!evaluateLValue(E->getSubExpr(), LVal)) return false; APValue RVal; // Note, we use the subexpression's type in order to retain cv-qualifiers. if (!handleLValueToRValueConversion(Info, E, E->getSubExpr()->getType(), LVal, RVal)) return InvalidBaseOK && evaluateLValueAsAllocSize(Info, LVal.Base, Result); return Success(RVal, E); } } return ExprEvaluatorBaseTy::VisitCastExpr(E); } static CharUnits GetAlignOfType(EvalInfo &Info, QualType T, UnaryExprOrTypeTrait ExprKind) { // C++ [expr.alignof]p3: // When alignof is applied to a reference type, the result is the // alignment of the referenced type. if (const ReferenceType *Ref = T->getAs()) T = Ref->getPointeeType(); if (T.getQualifiers().hasUnaligned()) return CharUnits::One(); const bool AlignOfReturnsPreferred = Info.Ctx.getLangOpts().getClangABICompat() <= LangOptions::ClangABI::Ver7; // __alignof is defined to return the preferred alignment. // Before 8, clang returned the preferred alignment for alignof and _Alignof // as well. if (ExprKind == UETT_PreferredAlignOf || AlignOfReturnsPreferred) return Info.Ctx.toCharUnitsFromBits( Info.Ctx.getPreferredTypeAlign(T.getTypePtr())); // alignof and _Alignof are defined to return the ABI alignment. else if (ExprKind == UETT_AlignOf) return Info.Ctx.getTypeAlignInChars(T.getTypePtr()); else llvm_unreachable("GetAlignOfType on a non-alignment ExprKind"); } static CharUnits GetAlignOfExpr(EvalInfo &Info, const Expr *E, UnaryExprOrTypeTrait ExprKind) { E = E->IgnoreParens(); // The kinds of expressions that we have special-case logic here for // should be kept up to date with the special checks for those // expressions in Sema. // alignof decl is always accepted, even if it doesn't make sense: we default // to 1 in those cases. if (const DeclRefExpr *DRE = dyn_cast(E)) return Info.Ctx.getDeclAlign(DRE->getDecl(), /*RefAsPointee*/true); if (const MemberExpr *ME = dyn_cast(E)) return Info.Ctx.getDeclAlign(ME->getMemberDecl(), /*RefAsPointee*/true); return GetAlignOfType(Info, E->getType(), ExprKind); } // To be clear: this happily visits unsupported builtins. Better name welcomed. bool PointerExprEvaluator::visitNonBuiltinCallExpr(const CallExpr *E) { if (ExprEvaluatorBaseTy::VisitCallExpr(E)) return true; if (!(InvalidBaseOK && getAllocSizeAttr(E))) return false; Result.setInvalid(E); QualType PointeeTy = E->getType()->castAs()->getPointeeType(); Result.addUnsizedArray(Info, E, PointeeTy); return true; } bool PointerExprEvaluator::VisitCallExpr(const CallExpr *E) { if (IsStringLiteralCall(E)) return Success(E); if (unsigned BuiltinOp = E->getBuiltinCallee()) return VisitBuiltinCallExpr(E, BuiltinOp); return visitNonBuiltinCallExpr(E); } bool PointerExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E, unsigned BuiltinOp) { switch (BuiltinOp) { case Builtin::BI__builtin_addressof: return evaluateLValue(E->getArg(0), Result); case Builtin::BI__builtin_assume_aligned: { // We need to be very careful here because: if the pointer does not have the // asserted alignment, then the behavior is undefined, and undefined // behavior is non-constant. if (!evaluatePointer(E->getArg(0), Result)) return false; LValue OffsetResult(Result); APSInt Alignment; if (!EvaluateInteger(E->getArg(1), Alignment, Info)) return false; CharUnits Align = CharUnits::fromQuantity(Alignment.getZExtValue()); if (E->getNumArgs() > 2) { APSInt Offset; if (!EvaluateInteger(E->getArg(2), Offset, Info)) return false; int64_t AdditionalOffset = -Offset.getZExtValue(); OffsetResult.Offset += CharUnits::fromQuantity(AdditionalOffset); } // If there is a base object, then it must have the correct alignment. if (OffsetResult.Base) { CharUnits BaseAlignment; if (const ValueDecl *VD = OffsetResult.Base.dyn_cast()) { BaseAlignment = Info.Ctx.getDeclAlign(VD); } else if (const Expr *E = OffsetResult.Base.dyn_cast()) { BaseAlignment = GetAlignOfExpr(Info, E, UETT_AlignOf); } else { BaseAlignment = GetAlignOfType( Info, OffsetResult.Base.getTypeInfoType(), UETT_AlignOf); } if (BaseAlignment < Align) { Result.Designator.setInvalid(); // FIXME: Add support to Diagnostic for long / long long. CCEDiag(E->getArg(0), diag::note_constexpr_baa_insufficient_alignment) << 0 << (unsigned)BaseAlignment.getQuantity() << (unsigned)Align.getQuantity(); return false; } } // The offset must also have the correct alignment. if (OffsetResult.Offset.alignTo(Align) != OffsetResult.Offset) { Result.Designator.setInvalid(); (OffsetResult.Base ? CCEDiag(E->getArg(0), diag::note_constexpr_baa_insufficient_alignment) << 1 : CCEDiag(E->getArg(0), diag::note_constexpr_baa_value_insufficient_alignment)) << (int)OffsetResult.Offset.getQuantity() << (unsigned)Align.getQuantity(); return false; } return true; } case Builtin::BI__builtin_launder: return evaluatePointer(E->getArg(0), Result); case Builtin::BIstrchr: case Builtin::BIwcschr: case Builtin::BImemchr: case Builtin::BIwmemchr: if (Info.getLangOpts().CPlusPlus11) Info.CCEDiag(E, diag::note_constexpr_invalid_function) << /*isConstexpr*/0 << /*isConstructor*/0 << (std::string("'") + Info.Ctx.BuiltinInfo.getName(BuiltinOp) + "'"); else Info.CCEDiag(E, diag::note_invalid_subexpr_in_const_expr); LLVM_FALLTHROUGH; case Builtin::BI__builtin_strchr: case Builtin::BI__builtin_wcschr: case Builtin::BI__builtin_memchr: case Builtin::BI__builtin_char_memchr: case Builtin::BI__builtin_wmemchr: { if (!Visit(E->getArg(0))) return false; APSInt Desired; if (!EvaluateInteger(E->getArg(1), Desired, Info)) return false; uint64_t MaxLength = uint64_t(-1); if (BuiltinOp != Builtin::BIstrchr && BuiltinOp != Builtin::BIwcschr && BuiltinOp != Builtin::BI__builtin_strchr && BuiltinOp != Builtin::BI__builtin_wcschr) { APSInt N; if (!EvaluateInteger(E->getArg(2), N, Info)) return false; MaxLength = N.getExtValue(); } // We cannot find the value if there are no candidates to match against. if (MaxLength == 0u) return ZeroInitialization(E); if (!Result.checkNullPointerForFoldAccess(Info, E, AK_Read) || Result.Designator.Invalid) return false; QualType CharTy = Result.Designator.getType(Info.Ctx); bool IsRawByte = BuiltinOp == Builtin::BImemchr || BuiltinOp == Builtin::BI__builtin_memchr; assert(IsRawByte || Info.Ctx.hasSameUnqualifiedType( CharTy, E->getArg(0)->getType()->getPointeeType())); // Pointers to const void may point to objects of incomplete type. if (IsRawByte && CharTy->isIncompleteType()) { Info.FFDiag(E, diag::note_constexpr_ltor_incomplete_type) << CharTy; return false; } // Give up on byte-oriented matching against multibyte elements. // FIXME: We can compare the bytes in the correct order. if (IsRawByte && Info.Ctx.getTypeSizeInChars(CharTy) != CharUnits::One()) return false; // Figure out what value we're actually looking for (after converting to // the corresponding unsigned type if necessary). uint64_t DesiredVal; bool StopAtNull = false; switch (BuiltinOp) { case Builtin::BIstrchr: case Builtin::BI__builtin_strchr: // strchr compares directly to the passed integer, and therefore // always fails if given an int that is not a char. if (!APSInt::isSameValue(HandleIntToIntCast(Info, E, CharTy, E->getArg(1)->getType(), Desired), Desired)) return ZeroInitialization(E); StopAtNull = true; LLVM_FALLTHROUGH; case Builtin::BImemchr: case Builtin::BI__builtin_memchr: case Builtin::BI__builtin_char_memchr: // memchr compares by converting both sides to unsigned char. That's also // correct for strchr if we get this far (to cope with plain char being // unsigned in the strchr case). DesiredVal = Desired.trunc(Info.Ctx.getCharWidth()).getZExtValue(); break; case Builtin::BIwcschr: case Builtin::BI__builtin_wcschr: StopAtNull = true; LLVM_FALLTHROUGH; case Builtin::BIwmemchr: case Builtin::BI__builtin_wmemchr: // wcschr and wmemchr are given a wchar_t to look for. Just use it. DesiredVal = Desired.getZExtValue(); break; } for (; MaxLength; --MaxLength) { APValue Char; if (!handleLValueToRValueConversion(Info, E, CharTy, Result, Char) || !Char.isInt()) return false; if (Char.getInt().getZExtValue() == DesiredVal) return true; if (StopAtNull && !Char.getInt()) break; if (!HandleLValueArrayAdjustment(Info, E, Result, CharTy, 1)) return false; } // Not found: return nullptr. return ZeroInitialization(E); } case Builtin::BImemcpy: case Builtin::BImemmove: case Builtin::BIwmemcpy: case Builtin::BIwmemmove: if (Info.getLangOpts().CPlusPlus11) Info.CCEDiag(E, diag::note_constexpr_invalid_function) << /*isConstexpr*/0 << /*isConstructor*/0 << (std::string("'") + Info.Ctx.BuiltinInfo.getName(BuiltinOp) + "'"); else Info.CCEDiag(E, diag::note_invalid_subexpr_in_const_expr); LLVM_FALLTHROUGH; case Builtin::BI__builtin_memcpy: case Builtin::BI__builtin_memmove: case Builtin::BI__builtin_wmemcpy: case Builtin::BI__builtin_wmemmove: { bool WChar = BuiltinOp == Builtin::BIwmemcpy || BuiltinOp == Builtin::BIwmemmove || BuiltinOp == Builtin::BI__builtin_wmemcpy || BuiltinOp == Builtin::BI__builtin_wmemmove; bool Move = BuiltinOp == Builtin::BImemmove || BuiltinOp == Builtin::BIwmemmove || BuiltinOp == Builtin::BI__builtin_memmove || BuiltinOp == Builtin::BI__builtin_wmemmove; // The result of mem* is the first argument. if (!Visit(E->getArg(0))) return false; LValue Dest = Result; LValue Src; if (!EvaluatePointer(E->getArg(1), Src, Info)) return false; APSInt N; if (!EvaluateInteger(E->getArg(2), N, Info)) return false; assert(!N.isSigned() && "memcpy and friends take an unsigned size"); // If the size is zero, we treat this as always being a valid no-op. // (Even if one of the src and dest pointers is null.) if (!N) return true; // Otherwise, if either of the operands is null, we can't proceed. Don't // try to determine the type of the copied objects, because there aren't // any. if (!Src.Base || !Dest.Base) { APValue Val; (!Src.Base ? Src : Dest).moveInto(Val); Info.FFDiag(E, diag::note_constexpr_memcpy_null) << Move << WChar << !!Src.Base << Val.getAsString(Info.Ctx, E->getArg(0)->getType()); return false; } if (Src.Designator.Invalid || Dest.Designator.Invalid) return false; // We require that Src and Dest are both pointers to arrays of // trivially-copyable type. (For the wide version, the designator will be // invalid if the designated object is not a wchar_t.) QualType T = Dest.Designator.getType(Info.Ctx); QualType SrcT = Src.Designator.getType(Info.Ctx); if (!Info.Ctx.hasSameUnqualifiedType(T, SrcT)) { Info.FFDiag(E, diag::note_constexpr_memcpy_type_pun) << Move << SrcT << T; return false; } if (T->isIncompleteType()) { Info.FFDiag(E, diag::note_constexpr_memcpy_incomplete_type) << Move << T; return false; } if (!T.isTriviallyCopyableType(Info.Ctx)) { Info.FFDiag(E, diag::note_constexpr_memcpy_nontrivial) << Move << T; return false; } // Figure out how many T's we're copying. uint64_t TSize = Info.Ctx.getTypeSizeInChars(T).getQuantity(); if (!WChar) { uint64_t Remainder; llvm::APInt OrigN = N; llvm::APInt::udivrem(OrigN, TSize, N, Remainder); if (Remainder) { Info.FFDiag(E, diag::note_constexpr_memcpy_unsupported) << Move << WChar << 0 << T << OrigN.toString(10, /*Signed*/false) << (unsigned)TSize; return false; } } // Check that the copying will remain within the arrays, just so that we // can give a more meaningful diagnostic. This implicitly also checks that // N fits into 64 bits. uint64_t RemainingSrcSize = Src.Designator.validIndexAdjustments().second; uint64_t RemainingDestSize = Dest.Designator.validIndexAdjustments().second; if (N.ugt(RemainingSrcSize) || N.ugt(RemainingDestSize)) { Info.FFDiag(E, diag::note_constexpr_memcpy_unsupported) << Move << WChar << (N.ugt(RemainingSrcSize) ? 1 : 2) << T << N.toString(10, /*Signed*/false); return false; } uint64_t NElems = N.getZExtValue(); uint64_t NBytes = NElems * TSize; // Check for overlap. int Direction = 1; if (HasSameBase(Src, Dest)) { uint64_t SrcOffset = Src.getLValueOffset().getQuantity(); uint64_t DestOffset = Dest.getLValueOffset().getQuantity(); if (DestOffset >= SrcOffset && DestOffset - SrcOffset < NBytes) { // Dest is inside the source region. if (!Move) { Info.FFDiag(E, diag::note_constexpr_memcpy_overlap) << WChar; return false; } // For memmove and friends, copy backwards. if (!HandleLValueArrayAdjustment(Info, E, Src, T, NElems - 1) || !HandleLValueArrayAdjustment(Info, E, Dest, T, NElems - 1)) return false; Direction = -1; } else if (!Move && SrcOffset >= DestOffset && SrcOffset - DestOffset < NBytes) { // Src is inside the destination region for memcpy: invalid. Info.FFDiag(E, diag::note_constexpr_memcpy_overlap) << WChar; return false; } } while (true) { APValue Val; if (!handleLValueToRValueConversion(Info, E, T, Src, Val) || !handleAssignment(Info, E, Dest, T, Val)) return false; // Do not iterate past the last element; if we're copying backwards, that // might take us off the start of the array. if (--NElems == 0) return true; if (!HandleLValueArrayAdjustment(Info, E, Src, T, Direction) || !HandleLValueArrayAdjustment(Info, E, Dest, T, Direction)) return false; } } default: return visitNonBuiltinCallExpr(E); } } //===----------------------------------------------------------------------===// // Member Pointer Evaluation //===----------------------------------------------------------------------===// namespace { class MemberPointerExprEvaluator : public ExprEvaluatorBase { MemberPtr &Result; bool Success(const ValueDecl *D) { Result = MemberPtr(D); return true; } public: MemberPointerExprEvaluator(EvalInfo &Info, MemberPtr &Result) : ExprEvaluatorBaseTy(Info), Result(Result) {} bool Success(const APValue &V, const Expr *E) { Result.setFrom(V); return true; } bool ZeroInitialization(const Expr *E) { return Success((const ValueDecl*)nullptr); } bool VisitCastExpr(const CastExpr *E); bool VisitUnaryAddrOf(const UnaryOperator *E); }; } // end anonymous namespace static bool EvaluateMemberPointer(const Expr *E, MemberPtr &Result, EvalInfo &Info) { assert(E->isRValue() && E->getType()->isMemberPointerType()); return MemberPointerExprEvaluator(Info, Result).Visit(E); } bool MemberPointerExprEvaluator::VisitCastExpr(const CastExpr *E) { switch (E->getCastKind()) { default: return ExprEvaluatorBaseTy::VisitCastExpr(E); case CK_NullToMemberPointer: VisitIgnoredValue(E->getSubExpr()); return ZeroInitialization(E); case CK_BaseToDerivedMemberPointer: { if (!Visit(E->getSubExpr())) return false; if (E->path_empty()) return true; // Base-to-derived member pointer casts store the path in derived-to-base // order, so iterate backwards. The CXXBaseSpecifier also provides us with // the wrong end of the derived->base arc, so stagger the path by one class. typedef std::reverse_iterator ReverseIter; for (ReverseIter PathI(E->path_end() - 1), PathE(E->path_begin()); PathI != PathE; ++PathI) { assert(!(*PathI)->isVirtual() && "memptr cast through vbase"); const CXXRecordDecl *Derived = (*PathI)->getType()->getAsCXXRecordDecl(); if (!Result.castToDerived(Derived)) return Error(E); } const Type *FinalTy = E->getType()->castAs()->getClass(); if (!Result.castToDerived(FinalTy->getAsCXXRecordDecl())) return Error(E); return true; } case CK_DerivedToBaseMemberPointer: if (!Visit(E->getSubExpr())) return false; for (CastExpr::path_const_iterator PathI = E->path_begin(), PathE = E->path_end(); PathI != PathE; ++PathI) { assert(!(*PathI)->isVirtual() && "memptr cast through vbase"); const CXXRecordDecl *Base = (*PathI)->getType()->getAsCXXRecordDecl(); if (!Result.castToBase(Base)) return Error(E); } return true; } } bool MemberPointerExprEvaluator::VisitUnaryAddrOf(const UnaryOperator *E) { // C++11 [expr.unary.op]p3 has very strict rules on how the address of a // member can be formed. return Success(cast(E->getSubExpr())->getDecl()); } //===----------------------------------------------------------------------===// // Record Evaluation //===----------------------------------------------------------------------===// namespace { class RecordExprEvaluator : public ExprEvaluatorBase { const LValue &This; APValue &Result; public: RecordExprEvaluator(EvalInfo &info, const LValue &This, APValue &Result) : ExprEvaluatorBaseTy(info), This(This), Result(Result) {} bool Success(const APValue &V, const Expr *E) { Result = V; return true; } bool ZeroInitialization(const Expr *E) { return ZeroInitialization(E, E->getType()); } bool ZeroInitialization(const Expr *E, QualType T); bool VisitCallExpr(const CallExpr *E) { return handleCallExpr(E, Result, &This); } bool VisitCastExpr(const CastExpr *E); bool VisitInitListExpr(const InitListExpr *E); bool VisitCXXConstructExpr(const CXXConstructExpr *E) { return VisitCXXConstructExpr(E, E->getType()); } bool VisitLambdaExpr(const LambdaExpr *E); bool VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *E); bool VisitCXXConstructExpr(const CXXConstructExpr *E, QualType T); bool VisitCXXStdInitializerListExpr(const CXXStdInitializerListExpr *E); bool VisitBinCmp(const BinaryOperator *E); }; } /// Perform zero-initialization on an object of non-union class type. /// C++11 [dcl.init]p5: /// To zero-initialize an object or reference of type T means: /// [...] /// -- if T is a (possibly cv-qualified) non-union class type, /// each non-static data member and each base-class subobject is /// zero-initialized static bool HandleClassZeroInitialization(EvalInfo &Info, const Expr *E, const RecordDecl *RD, const LValue &This, APValue &Result) { assert(!RD->isUnion() && "Expected non-union class type"); const CXXRecordDecl *CD = dyn_cast(RD); Result = APValue(APValue::UninitStruct(), CD ? CD->getNumBases() : 0, std::distance(RD->field_begin(), RD->field_end())); if (RD->isInvalidDecl()) return false; const ASTRecordLayout &Layout = Info.Ctx.getASTRecordLayout(RD); if (CD) { unsigned Index = 0; for (CXXRecordDecl::base_class_const_iterator I = CD->bases_begin(), End = CD->bases_end(); I != End; ++I, ++Index) { const CXXRecordDecl *Base = I->getType()->getAsCXXRecordDecl(); LValue Subobject = This; if (!HandleLValueDirectBase(Info, E, Subobject, CD, Base, &Layout)) return false; if (!HandleClassZeroInitialization(Info, E, Base, Subobject, Result.getStructBase(Index))) return false; } } for (const auto *I : RD->fields()) { // -- if T is a reference type, no initialization is performed. if (I->getType()->isReferenceType()) continue; LValue Subobject = This; if (!HandleLValueMember(Info, E, Subobject, I, &Layout)) return false; ImplicitValueInitExpr VIE(I->getType()); if (!EvaluateInPlace( Result.getStructField(I->getFieldIndex()), Info, Subobject, &VIE)) return false; } return true; } bool RecordExprEvaluator::ZeroInitialization(const Expr *E, QualType T) { const RecordDecl *RD = T->castAs()->getDecl(); if (RD->isInvalidDecl()) return false; if (RD->isUnion()) { // C++11 [dcl.init]p5: If T is a (possibly cv-qualified) union type, the // object's first non-static named data member is zero-initialized RecordDecl::field_iterator I = RD->field_begin(); if (I == RD->field_end()) { Result = APValue((const FieldDecl*)nullptr); return true; } LValue Subobject = This; if (!HandleLValueMember(Info, E, Subobject, *I)) return false; Result = APValue(*I); ImplicitValueInitExpr VIE(I->getType()); return EvaluateInPlace(Result.getUnionValue(), Info, Subobject, &VIE); } if (isa(RD) && cast(RD)->getNumVBases()) { Info.FFDiag(E, diag::note_constexpr_virtual_base) << RD; return false; } return HandleClassZeroInitialization(Info, E, RD, This, Result); } bool RecordExprEvaluator::VisitCastExpr(const CastExpr *E) { switch (E->getCastKind()) { default: return ExprEvaluatorBaseTy::VisitCastExpr(E); case CK_ConstructorConversion: return Visit(E->getSubExpr()); case CK_DerivedToBase: case CK_UncheckedDerivedToBase: { APValue DerivedObject; if (!Evaluate(DerivedObject, Info, E->getSubExpr())) return false; if (!DerivedObject.isStruct()) return Error(E->getSubExpr()); // Derived-to-base rvalue conversion: just slice off the derived part. APValue *Value = &DerivedObject; const CXXRecordDecl *RD = E->getSubExpr()->getType()->getAsCXXRecordDecl(); for (CastExpr::path_const_iterator PathI = E->path_begin(), PathE = E->path_end(); PathI != PathE; ++PathI) { assert(!(*PathI)->isVirtual() && "record rvalue with virtual base"); const CXXRecordDecl *Base = (*PathI)->getType()->getAsCXXRecordDecl(); Value = &Value->getStructBase(getBaseIndex(RD, Base)); RD = Base; } Result = *Value; return true; } } } bool RecordExprEvaluator::VisitInitListExpr(const InitListExpr *E) { if (E->isTransparent()) return Visit(E->getInit(0)); const RecordDecl *RD = E->getType()->castAs()->getDecl(); if (RD->isInvalidDecl()) return false; const ASTRecordLayout &Layout = Info.Ctx.getASTRecordLayout(RD); auto *CXXRD = dyn_cast(RD); EvalInfo::EvaluatingConstructorRAII EvalObj( Info, ObjectUnderConstruction{This.getLValueBase(), This.Designator.Entries}, CXXRD && CXXRD->getNumBases()); if (RD->isUnion()) { const FieldDecl *Field = E->getInitializedFieldInUnion(); Result = APValue(Field); if (!Field) return true; // If the initializer list for a union does not contain any elements, the // first element of the union is value-initialized. // FIXME: The element should be initialized from an initializer list. // Is this difference ever observable for initializer lists which // we don't build? ImplicitValueInitExpr VIE(Field->getType()); const Expr *InitExpr = E->getNumInits() ? E->getInit(0) : &VIE; LValue Subobject = This; if (!HandleLValueMember(Info, InitExpr, Subobject, Field, &Layout)) return false; // Temporarily override This, in case there's a CXXDefaultInitExpr in here. ThisOverrideRAII ThisOverride(*Info.CurrentCall, &This, isa(InitExpr)); return EvaluateInPlace(Result.getUnionValue(), Info, Subobject, InitExpr); } if (!Result.hasValue()) Result = APValue(APValue::UninitStruct(), CXXRD ? CXXRD->getNumBases() : 0, std::distance(RD->field_begin(), RD->field_end())); unsigned ElementNo = 0; bool Success = true; // Initialize base classes. if (CXXRD && CXXRD->getNumBases()) { for (const auto &Base : CXXRD->bases()) { assert(ElementNo < E->getNumInits() && "missing init for base class"); const Expr *Init = E->getInit(ElementNo); LValue Subobject = This; if (!HandleLValueBase(Info, Init, Subobject, CXXRD, &Base)) return false; APValue &FieldVal = Result.getStructBase(ElementNo); if (!EvaluateInPlace(FieldVal, Info, Subobject, Init)) { if (!Info.noteFailure()) return false; Success = false; } ++ElementNo; } EvalObj.finishedConstructingBases(); } // Initialize members. for (const auto *Field : RD->fields()) { // Anonymous bit-fields are not considered members of the class for // purposes of aggregate initialization. if (Field->isUnnamedBitfield()) continue; LValue Subobject = This; bool HaveInit = ElementNo < E->getNumInits(); // FIXME: Diagnostics here should point to the end of the initializer // list, not the start. if (!HandleLValueMember(Info, HaveInit ? E->getInit(ElementNo) : E, Subobject, Field, &Layout)) return false; // Perform an implicit value-initialization for members beyond the end of // the initializer list. ImplicitValueInitExpr VIE(HaveInit ? Info.Ctx.IntTy : Field->getType()); const Expr *Init = HaveInit ? E->getInit(ElementNo++) : &VIE; // Temporarily override This, in case there's a CXXDefaultInitExpr in here. ThisOverrideRAII ThisOverride(*Info.CurrentCall, &This, isa(Init)); APValue &FieldVal = Result.getStructField(Field->getFieldIndex()); if (!EvaluateInPlace(FieldVal, Info, Subobject, Init) || (Field->isBitField() && !truncateBitfieldValue(Info, Init, FieldVal, Field))) { if (!Info.noteFailure()) return false; Success = false; } } return Success; } bool RecordExprEvaluator::VisitCXXConstructExpr(const CXXConstructExpr *E, QualType T) { // Note that E's type is not necessarily the type of our class here; we might // be initializing an array element instead. const CXXConstructorDecl *FD = E->getConstructor(); if (FD->isInvalidDecl() || FD->getParent()->isInvalidDecl()) return false; bool ZeroInit = E->requiresZeroInitialization(); if (CheckTrivialDefaultConstructor(Info, E->getExprLoc(), FD, ZeroInit)) { // If we've already performed zero-initialization, we're already done. if (Result.hasValue()) return true; // We can get here in two different ways: // 1) We're performing value-initialization, and should zero-initialize // the object, or // 2) We're performing default-initialization of an object with a trivial // constexpr default constructor, in which case we should start the // lifetimes of all the base subobjects (there can be no data member // subobjects in this case) per [basic.life]p1. // Either way, ZeroInitialization is appropriate. return ZeroInitialization(E, T); } const FunctionDecl *Definition = nullptr; auto Body = FD->getBody(Definition); if (!CheckConstexprFunction(Info, E->getExprLoc(), FD, Definition, Body)) return false; // Avoid materializing a temporary for an elidable copy/move constructor. if (E->isElidable() && !ZeroInit) if (const MaterializeTemporaryExpr *ME = dyn_cast(E->getArg(0))) return Visit(ME->GetTemporaryExpr()); if (ZeroInit && !ZeroInitialization(E, T)) return false; auto Args = llvm::makeArrayRef(E->getArgs(), E->getNumArgs()); return HandleConstructorCall(E, This, Args, cast(Definition), Info, Result); } bool RecordExprEvaluator::VisitCXXInheritedCtorInitExpr( const CXXInheritedCtorInitExpr *E) { if (!Info.CurrentCall) { assert(Info.checkingPotentialConstantExpression()); return false; } const CXXConstructorDecl *FD = E->getConstructor(); if (FD->isInvalidDecl() || FD->getParent()->isInvalidDecl()) return false; const FunctionDecl *Definition = nullptr; auto Body = FD->getBody(Definition); if (!CheckConstexprFunction(Info, E->getExprLoc(), FD, Definition, Body)) return false; return HandleConstructorCall(E, This, Info.CurrentCall->Arguments, cast(Definition), Info, Result); } bool RecordExprEvaluator::VisitCXXStdInitializerListExpr( const CXXStdInitializerListExpr *E) { const ConstantArrayType *ArrayType = Info.Ctx.getAsConstantArrayType(E->getSubExpr()->getType()); LValue Array; if (!EvaluateLValue(E->getSubExpr(), Array, Info)) return false; // Get a pointer to the first element of the array. Array.addArray(Info, E, ArrayType); // FIXME: Perform the checks on the field types in SemaInit. RecordDecl *Record = E->getType()->castAs()->getDecl(); RecordDecl::field_iterator Field = Record->field_begin(); if (Field == Record->field_end()) return Error(E); // Start pointer. if (!Field->getType()->isPointerType() || !Info.Ctx.hasSameType(Field->getType()->getPointeeType(), ArrayType->getElementType())) return Error(E); // FIXME: What if the initializer_list type has base classes, etc? Result = APValue(APValue::UninitStruct(), 0, 2); Array.moveInto(Result.getStructField(0)); if (++Field == Record->field_end()) return Error(E); if (Field->getType()->isPointerType() && Info.Ctx.hasSameType(Field->getType()->getPointeeType(), ArrayType->getElementType())) { // End pointer. if (!HandleLValueArrayAdjustment(Info, E, Array, ArrayType->getElementType(), ArrayType->getSize().getZExtValue())) return false; Array.moveInto(Result.getStructField(1)); } else if (Info.Ctx.hasSameType(Field->getType(), Info.Ctx.getSizeType())) // Length. Result.getStructField(1) = APValue(APSInt(ArrayType->getSize())); else return Error(E); if (++Field != Record->field_end()) return Error(E); return true; } bool RecordExprEvaluator::VisitLambdaExpr(const LambdaExpr *E) { const CXXRecordDecl *ClosureClass = E->getLambdaClass(); if (ClosureClass->isInvalidDecl()) return false; if (Info.checkingPotentialConstantExpression()) return true; const size_t NumFields = std::distance(ClosureClass->field_begin(), ClosureClass->field_end()); assert(NumFields == (size_t)std::distance(E->capture_init_begin(), E->capture_init_end()) && "The number of lambda capture initializers should equal the number of " "fields within the closure type"); Result = APValue(APValue::UninitStruct(), /*NumBases*/0, NumFields); // Iterate through all the lambda's closure object's fields and initialize // them. auto *CaptureInitIt = E->capture_init_begin(); const LambdaCapture *CaptureIt = ClosureClass->captures_begin(); bool Success = true; for (const auto *Field : ClosureClass->fields()) { assert(CaptureInitIt != E->capture_init_end()); // Get the initializer for this field Expr *const CurFieldInit = *CaptureInitIt++; // If there is no initializer, either this is a VLA or an error has // occurred. if (!CurFieldInit) return Error(E); APValue &FieldVal = Result.getStructField(Field->getFieldIndex()); if (!EvaluateInPlace(FieldVal, Info, This, CurFieldInit)) { if (!Info.keepEvaluatingAfterFailure()) return false; Success = false; } ++CaptureIt; } return Success; } static bool EvaluateRecord(const Expr *E, const LValue &This, APValue &Result, EvalInfo &Info) { assert(E->isRValue() && E->getType()->isRecordType() && "can't evaluate expression as a record rvalue"); return RecordExprEvaluator(Info, This, Result).Visit(E); } //===----------------------------------------------------------------------===// // Temporary Evaluation // // Temporaries are represented in the AST as rvalues, but generally behave like // lvalues. The full-object of which the temporary is a subobject is implicitly // materialized so that a reference can bind to it. //===----------------------------------------------------------------------===// namespace { class TemporaryExprEvaluator : public LValueExprEvaluatorBase { public: TemporaryExprEvaluator(EvalInfo &Info, LValue &Result) : LValueExprEvaluatorBaseTy(Info, Result, false) {} /// Visit an expression which constructs the value of this temporary. bool VisitConstructExpr(const Expr *E) { APValue &Value = createTemporary(E, false, Result, *Info.CurrentCall); return EvaluateInPlace(Value, Info, Result, E); } bool VisitCastExpr(const CastExpr *E) { switch (E->getCastKind()) { default: return LValueExprEvaluatorBaseTy::VisitCastExpr(E); case CK_ConstructorConversion: return VisitConstructExpr(E->getSubExpr()); } } bool VisitInitListExpr(const InitListExpr *E) { return VisitConstructExpr(E); } bool VisitCXXConstructExpr(const CXXConstructExpr *E) { return VisitConstructExpr(E); } bool VisitCallExpr(const CallExpr *E) { return VisitConstructExpr(E); } bool VisitCXXStdInitializerListExpr(const CXXStdInitializerListExpr *E) { return VisitConstructExpr(E); } bool VisitLambdaExpr(const LambdaExpr *E) { return VisitConstructExpr(E); } }; } // end anonymous namespace /// Evaluate an expression of record type as a temporary. static bool EvaluateTemporary(const Expr *E, LValue &Result, EvalInfo &Info) { assert(E->isRValue() && E->getType()->isRecordType()); return TemporaryExprEvaluator(Info, Result).Visit(E); } //===----------------------------------------------------------------------===// // Vector Evaluation //===----------------------------------------------------------------------===// namespace { class VectorExprEvaluator : public ExprEvaluatorBase { APValue &Result; public: VectorExprEvaluator(EvalInfo &info, APValue &Result) : ExprEvaluatorBaseTy(info), Result(Result) {} bool Success(ArrayRef V, const Expr *E) { assert(V.size() == E->getType()->castAs()->getNumElements()); // FIXME: remove this APValue copy. Result = APValue(V.data(), V.size()); return true; } bool Success(const APValue &V, const Expr *E) { assert(V.isVector()); Result = V; return true; } bool ZeroInitialization(const Expr *E); bool VisitUnaryReal(const UnaryOperator *E) { return Visit(E->getSubExpr()); } bool VisitCastExpr(const CastExpr* E); bool VisitInitListExpr(const InitListExpr *E); bool VisitUnaryImag(const UnaryOperator *E); // FIXME: Missing: unary -, unary ~, binary add/sub/mul/div, // binary comparisons, binary and/or/xor, // shufflevector, ExtVectorElementExpr }; } // end anonymous namespace static bool EvaluateVector(const Expr* E, APValue& Result, EvalInfo &Info) { assert(E->isRValue() && E->getType()->isVectorType() &&"not a vector rvalue"); return VectorExprEvaluator(Info, Result).Visit(E); } bool VectorExprEvaluator::VisitCastExpr(const CastExpr *E) { const VectorType *VTy = E->getType()->castAs(); unsigned NElts = VTy->getNumElements(); const Expr *SE = E->getSubExpr(); QualType SETy = SE->getType(); switch (E->getCastKind()) { case CK_VectorSplat: { APValue Val = APValue(); if (SETy->isIntegerType()) { APSInt IntResult; if (!EvaluateInteger(SE, IntResult, Info)) return false; Val = APValue(std::move(IntResult)); } else if (SETy->isRealFloatingType()) { APFloat FloatResult(0.0); if (!EvaluateFloat(SE, FloatResult, Info)) return false; Val = APValue(std::move(FloatResult)); } else { return Error(E); } // Splat and create vector APValue. SmallVector Elts(NElts, Val); return Success(Elts, E); } case CK_BitCast: { // Evaluate the operand into an APInt we can extract from. llvm::APInt SValInt; if (!EvalAndBitcastToAPInt(Info, SE, SValInt)) return false; // Extract the elements QualType EltTy = VTy->getElementType(); unsigned EltSize = Info.Ctx.getTypeSize(EltTy); bool BigEndian = Info.Ctx.getTargetInfo().isBigEndian(); SmallVector Elts; if (EltTy->isRealFloatingType()) { const llvm::fltSemantics &Sem = Info.Ctx.getFloatTypeSemantics(EltTy); unsigned FloatEltSize = EltSize; if (&Sem == &APFloat::x87DoubleExtended()) FloatEltSize = 80; for (unsigned i = 0; i < NElts; i++) { llvm::APInt Elt; if (BigEndian) Elt = SValInt.rotl(i*EltSize+FloatEltSize).trunc(FloatEltSize); else Elt = SValInt.rotr(i*EltSize).trunc(FloatEltSize); Elts.push_back(APValue(APFloat(Sem, Elt))); } } else if (EltTy->isIntegerType()) { for (unsigned i = 0; i < NElts; i++) { llvm::APInt Elt; if (BigEndian) Elt = SValInt.rotl(i*EltSize+EltSize).zextOrTrunc(EltSize); else Elt = SValInt.rotr(i*EltSize).zextOrTrunc(EltSize); Elts.push_back(APValue(APSInt(Elt, EltTy->isSignedIntegerType()))); } } else { return Error(E); } return Success(Elts, E); } default: return ExprEvaluatorBaseTy::VisitCastExpr(E); } } bool VectorExprEvaluator::VisitInitListExpr(const InitListExpr *E) { const VectorType *VT = E->getType()->castAs(); unsigned NumInits = E->getNumInits(); unsigned NumElements = VT->getNumElements(); QualType EltTy = VT->getElementType(); SmallVector Elements; // The number of initializers can be less than the number of // vector elements. For OpenCL, this can be due to nested vector // initialization. For GCC compatibility, missing trailing elements // should be initialized with zeroes. unsigned CountInits = 0, CountElts = 0; while (CountElts < NumElements) { // Handle nested vector initialization. if (CountInits < NumInits && E->getInit(CountInits)->getType()->isVectorType()) { APValue v; if (!EvaluateVector(E->getInit(CountInits), v, Info)) return Error(E); unsigned vlen = v.getVectorLength(); for (unsigned j = 0; j < vlen; j++) Elements.push_back(v.getVectorElt(j)); CountElts += vlen; } else if (EltTy->isIntegerType()) { llvm::APSInt sInt(32); if (CountInits < NumInits) { if (!EvaluateInteger(E->getInit(CountInits), sInt, Info)) return false; } else // trailing integer zero. sInt = Info.Ctx.MakeIntValue(0, EltTy); Elements.push_back(APValue(sInt)); CountElts++; } else { llvm::APFloat f(0.0); if (CountInits < NumInits) { if (!EvaluateFloat(E->getInit(CountInits), f, Info)) return false; } else // trailing float zero. f = APFloat::getZero(Info.Ctx.getFloatTypeSemantics(EltTy)); Elements.push_back(APValue(f)); CountElts++; } CountInits++; } return Success(Elements, E); } bool VectorExprEvaluator::ZeroInitialization(const Expr *E) { const VectorType *VT = E->getType()->getAs(); QualType EltTy = VT->getElementType(); APValue ZeroElement; if (EltTy->isIntegerType()) ZeroElement = APValue(Info.Ctx.MakeIntValue(0, EltTy)); else ZeroElement = APValue(APFloat::getZero(Info.Ctx.getFloatTypeSemantics(EltTy))); SmallVector Elements(VT->getNumElements(), ZeroElement); return Success(Elements, E); } bool VectorExprEvaluator::VisitUnaryImag(const UnaryOperator *E) { VisitIgnoredValue(E->getSubExpr()); return ZeroInitialization(E); } //===----------------------------------------------------------------------===// // Array Evaluation //===----------------------------------------------------------------------===// namespace { class ArrayExprEvaluator : public ExprEvaluatorBase { const LValue &This; APValue &Result; public: ArrayExprEvaluator(EvalInfo &Info, const LValue &This, APValue &Result) : ExprEvaluatorBaseTy(Info), This(This), Result(Result) {} bool Success(const APValue &V, const Expr *E) { assert(V.isArray() && "expected array"); Result = V; return true; } bool ZeroInitialization(const Expr *E) { const ConstantArrayType *CAT = Info.Ctx.getAsConstantArrayType(E->getType()); if (!CAT) return Error(E); Result = APValue(APValue::UninitArray(), 0, CAT->getSize().getZExtValue()); if (!Result.hasArrayFiller()) return true; // Zero-initialize all elements. LValue Subobject = This; Subobject.addArray(Info, E, CAT); ImplicitValueInitExpr VIE(CAT->getElementType()); return EvaluateInPlace(Result.getArrayFiller(), Info, Subobject, &VIE); } bool VisitCallExpr(const CallExpr *E) { return handleCallExpr(E, Result, &This); } bool VisitInitListExpr(const InitListExpr *E); bool VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E); bool VisitCXXConstructExpr(const CXXConstructExpr *E); bool VisitCXXConstructExpr(const CXXConstructExpr *E, const LValue &Subobject, APValue *Value, QualType Type); bool VisitStringLiteral(const StringLiteral *E) { expandStringLiteral(Info, E, Result); return true; } }; } // end anonymous namespace static bool EvaluateArray(const Expr *E, const LValue &This, APValue &Result, EvalInfo &Info) { assert(E->isRValue() && E->getType()->isArrayType() && "not an array rvalue"); return ArrayExprEvaluator(Info, This, Result).Visit(E); } // Return true iff the given array filler may depend on the element index. static bool MaybeElementDependentArrayFiller(const Expr *FillerExpr) { // For now, just whitelist non-class value-initialization and initialization // lists comprised of them. if (isa(FillerExpr)) return false; if (const InitListExpr *ILE = dyn_cast(FillerExpr)) { for (unsigned I = 0, E = ILE->getNumInits(); I != E; ++I) { if (MaybeElementDependentArrayFiller(ILE->getInit(I))) return true; } return false; } return true; } bool ArrayExprEvaluator::VisitInitListExpr(const InitListExpr *E) { const ConstantArrayType *CAT = Info.Ctx.getAsConstantArrayType(E->getType()); if (!CAT) return Error(E); // C++11 [dcl.init.string]p1: A char array [...] can be initialized by [...] // an appropriately-typed string literal enclosed in braces. if (E->isStringLiteralInit()) return Visit(E->getInit(0)); bool Success = true; assert((!Result.isArray() || Result.getArrayInitializedElts() == 0) && "zero-initialized array shouldn't have any initialized elts"); APValue Filler; if (Result.isArray() && Result.hasArrayFiller()) Filler = Result.getArrayFiller(); unsigned NumEltsToInit = E->getNumInits(); unsigned NumElts = CAT->getSize().getZExtValue(); const Expr *FillerExpr = E->hasArrayFiller() ? E->getArrayFiller() : nullptr; // If the initializer might depend on the array index, run it for each // array element. if (NumEltsToInit != NumElts && MaybeElementDependentArrayFiller(FillerExpr)) NumEltsToInit = NumElts; LLVM_DEBUG(llvm::dbgs() << "The number of elements to initialize: " << NumEltsToInit << ".\n"); Result = APValue(APValue::UninitArray(), NumEltsToInit, NumElts); // If the array was previously zero-initialized, preserve the // zero-initialized values. if (Filler.hasValue()) { for (unsigned I = 0, E = Result.getArrayInitializedElts(); I != E; ++I) Result.getArrayInitializedElt(I) = Filler; if (Result.hasArrayFiller()) Result.getArrayFiller() = Filler; } LValue Subobject = This; Subobject.addArray(Info, E, CAT); for (unsigned Index = 0; Index != NumEltsToInit; ++Index) { const Expr *Init = Index < E->getNumInits() ? E->getInit(Index) : FillerExpr; if (!EvaluateInPlace(Result.getArrayInitializedElt(Index), Info, Subobject, Init) || !HandleLValueArrayAdjustment(Info, Init, Subobject, CAT->getElementType(), 1)) { if (!Info.noteFailure()) return false; Success = false; } } if (!Result.hasArrayFiller()) return Success; // If we get here, we have a trivial filler, which we can just evaluate // once and splat over the rest of the array elements. assert(FillerExpr && "no array filler for incomplete init list"); return EvaluateInPlace(Result.getArrayFiller(), Info, Subobject, FillerExpr) && Success; } bool ArrayExprEvaluator::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E) { if (E->getCommonExpr() && !Evaluate(Info.CurrentCall->createTemporary(E->getCommonExpr(), false), Info, E->getCommonExpr()->getSourceExpr())) return false; auto *CAT = cast(E->getType()->castAsArrayTypeUnsafe()); uint64_t Elements = CAT->getSize().getZExtValue(); Result = APValue(APValue::UninitArray(), Elements, Elements); LValue Subobject = This; Subobject.addArray(Info, E, CAT); bool Success = true; for (EvalInfo::ArrayInitLoopIndex Index(Info); Index != Elements; ++Index) { if (!EvaluateInPlace(Result.getArrayInitializedElt(Index), Info, Subobject, E->getSubExpr()) || !HandleLValueArrayAdjustment(Info, E, Subobject, CAT->getElementType(), 1)) { if (!Info.noteFailure()) return false; Success = false; } } return Success; } bool ArrayExprEvaluator::VisitCXXConstructExpr(const CXXConstructExpr *E) { return VisitCXXConstructExpr(E, This, &Result, E->getType()); } bool ArrayExprEvaluator::VisitCXXConstructExpr(const CXXConstructExpr *E, const LValue &Subobject, APValue *Value, QualType Type) { bool HadZeroInit = Value->hasValue(); if (const ConstantArrayType *CAT = Info.Ctx.getAsConstantArrayType(Type)) { unsigned N = CAT->getSize().getZExtValue(); // Preserve the array filler if we had prior zero-initialization. APValue Filler = HadZeroInit && Value->hasArrayFiller() ? Value->getArrayFiller() : APValue(); *Value = APValue(APValue::UninitArray(), N, N); if (HadZeroInit) for (unsigned I = 0; I != N; ++I) Value->getArrayInitializedElt(I) = Filler; // Initialize the elements. LValue ArrayElt = Subobject; ArrayElt.addArray(Info, E, CAT); for (unsigned I = 0; I != N; ++I) if (!VisitCXXConstructExpr(E, ArrayElt, &Value->getArrayInitializedElt(I), CAT->getElementType()) || !HandleLValueArrayAdjustment(Info, E, ArrayElt, CAT->getElementType(), 1)) return false; return true; } if (!Type->isRecordType()) return Error(E); return RecordExprEvaluator(Info, Subobject, *Value) .VisitCXXConstructExpr(E, Type); } //===----------------------------------------------------------------------===// // Integer Evaluation // // As a GNU extension, we support casting pointers to sufficiently-wide integer // types and back in constant folding. Integer values are thus represented // either as an integer-valued APValue, or as an lvalue-valued APValue. //===----------------------------------------------------------------------===// namespace { class IntExprEvaluator : public ExprEvaluatorBase { APValue &Result; public: IntExprEvaluator(EvalInfo &info, APValue &result) : ExprEvaluatorBaseTy(info), Result(result) {} bool Success(const llvm::APSInt &SI, const Expr *E, APValue &Result) { assert(E->getType()->isIntegralOrEnumerationType() && "Invalid evaluation result."); assert(SI.isSigned() == E->getType()->isSignedIntegerOrEnumerationType() && "Invalid evaluation result."); assert(SI.getBitWidth() == Info.Ctx.getIntWidth(E->getType()) && "Invalid evaluation result."); Result = APValue(SI); return true; } bool Success(const llvm::APSInt &SI, const Expr *E) { return Success(SI, E, Result); } bool Success(const llvm::APInt &I, const Expr *E, APValue &Result) { assert(E->getType()->isIntegralOrEnumerationType() && "Invalid evaluation result."); assert(I.getBitWidth() == Info.Ctx.getIntWidth(E->getType()) && "Invalid evaluation result."); Result = APValue(APSInt(I)); Result.getInt().setIsUnsigned( E->getType()->isUnsignedIntegerOrEnumerationType()); return true; } bool Success(const llvm::APInt &I, const Expr *E) { return Success(I, E, Result); } bool Success(uint64_t Value, const Expr *E, APValue &Result) { assert(E->getType()->isIntegralOrEnumerationType() && "Invalid evaluation result."); Result = APValue(Info.Ctx.MakeIntValue(Value, E->getType())); return true; } bool Success(uint64_t Value, const Expr *E) { return Success(Value, E, Result); } bool Success(CharUnits Size, const Expr *E) { return Success(Size.getQuantity(), E); } bool Success(const APValue &V, const Expr *E) { if (V.isLValue() || V.isAddrLabelDiff() || V.isIndeterminate()) { Result = V; return true; } return Success(V.getInt(), E); } bool ZeroInitialization(const Expr *E) { return Success(0, E); } //===--------------------------------------------------------------------===// // Visitor Methods //===--------------------------------------------------------------------===// bool VisitConstantExpr(const ConstantExpr *E); bool VisitIntegerLiteral(const IntegerLiteral *E) { return Success(E->getValue(), E); } bool VisitCharacterLiteral(const CharacterLiteral *E) { return Success(E->getValue(), E); } bool CheckReferencedDecl(const Expr *E, const Decl *D); bool VisitDeclRefExpr(const DeclRefExpr *E) { if (CheckReferencedDecl(E, E->getDecl())) return true; return ExprEvaluatorBaseTy::VisitDeclRefExpr(E); } bool VisitMemberExpr(const MemberExpr *E) { if (CheckReferencedDecl(E, E->getMemberDecl())) { VisitIgnoredBaseExpression(E->getBase()); return true; } return ExprEvaluatorBaseTy::VisitMemberExpr(E); } bool VisitCallExpr(const CallExpr *E); bool VisitBuiltinCallExpr(const CallExpr *E, unsigned BuiltinOp); bool VisitBinaryOperator(const BinaryOperator *E); bool VisitOffsetOfExpr(const OffsetOfExpr *E); bool VisitUnaryOperator(const UnaryOperator *E); bool VisitCastExpr(const CastExpr* E); bool VisitUnaryExprOrTypeTraitExpr(const UnaryExprOrTypeTraitExpr *E); bool VisitCXXBoolLiteralExpr(const CXXBoolLiteralExpr *E) { return Success(E->getValue(), E); } bool VisitObjCBoolLiteralExpr(const ObjCBoolLiteralExpr *E) { return Success(E->getValue(), E); } bool VisitArrayInitIndexExpr(const ArrayInitIndexExpr *E) { if (Info.ArrayInitIndex == uint64_t(-1)) { // We were asked to evaluate this subexpression independent of the // enclosing ArrayInitLoopExpr. We can't do that. Info.FFDiag(E); return false; } return Success(Info.ArrayInitIndex, E); } // Note, GNU defines __null as an integer, not a pointer. bool VisitGNUNullExpr(const GNUNullExpr *E) { return ZeroInitialization(E); } bool VisitTypeTraitExpr(const TypeTraitExpr *E) { return Success(E->getValue(), E); } bool VisitArrayTypeTraitExpr(const ArrayTypeTraitExpr *E) { return Success(E->getValue(), E); } bool VisitExpressionTraitExpr(const ExpressionTraitExpr *E) { return Success(E->getValue(), E); } bool VisitUnaryReal(const UnaryOperator *E); bool VisitUnaryImag(const UnaryOperator *E); bool VisitCXXNoexceptExpr(const CXXNoexceptExpr *E); bool VisitSizeOfPackExpr(const SizeOfPackExpr *E); bool VisitSourceLocExpr(const SourceLocExpr *E); // FIXME: Missing: array subscript of vector, member of vector }; class FixedPointExprEvaluator : public ExprEvaluatorBase { APValue &Result; public: FixedPointExprEvaluator(EvalInfo &info, APValue &result) : ExprEvaluatorBaseTy(info), Result(result) {} bool Success(const llvm::APInt &I, const Expr *E) { return Success( APFixedPoint(I, Info.Ctx.getFixedPointSemantics(E->getType())), E); } bool Success(uint64_t Value, const Expr *E) { return Success( APFixedPoint(Value, Info.Ctx.getFixedPointSemantics(E->getType())), E); } bool Success(const APValue &V, const Expr *E) { return Success(V.getFixedPoint(), E); } bool Success(const APFixedPoint &V, const Expr *E) { assert(E->getType()->isFixedPointType() && "Invalid evaluation result."); assert(V.getWidth() == Info.Ctx.getIntWidth(E->getType()) && "Invalid evaluation result."); Result = APValue(V); return true; } //===--------------------------------------------------------------------===// // Visitor Methods //===--------------------------------------------------------------------===// bool VisitFixedPointLiteral(const FixedPointLiteral *E) { return Success(E->getValue(), E); } bool VisitCastExpr(const CastExpr *E); bool VisitUnaryOperator(const UnaryOperator *E); bool VisitBinaryOperator(const BinaryOperator *E); }; } // end anonymous namespace /// EvaluateIntegerOrLValue - Evaluate an rvalue integral-typed expression, and /// produce either the integer value or a pointer. /// /// GCC has a heinous extension which folds casts between pointer types and /// pointer-sized integral types. We support this by allowing the evaluation of /// an integer rvalue to produce a pointer (represented as an lvalue) instead. /// Some simple arithmetic on such values is supported (they are treated much /// like char*). static bool EvaluateIntegerOrLValue(const Expr *E, APValue &Result, EvalInfo &Info) { assert(E->isRValue() && E->getType()->isIntegralOrEnumerationType()); return IntExprEvaluator(Info, Result).Visit(E); } static bool EvaluateInteger(const Expr *E, APSInt &Result, EvalInfo &Info) { APValue Val; if (!EvaluateIntegerOrLValue(E, Val, Info)) return false; if (!Val.isInt()) { // FIXME: It would be better to produce the diagnostic for casting // a pointer to an integer. Info.FFDiag(E, diag::note_invalid_subexpr_in_const_expr); return false; } Result = Val.getInt(); return true; } bool IntExprEvaluator::VisitSourceLocExpr(const SourceLocExpr *E) { APValue Evaluated = E->EvaluateInContext( Info.Ctx, Info.CurrentCall->CurSourceLocExprScope.getDefaultExpr()); return Success(Evaluated, E); } static bool EvaluateFixedPoint(const Expr *E, APFixedPoint &Result, EvalInfo &Info) { if (E->getType()->isFixedPointType()) { APValue Val; if (!FixedPointExprEvaluator(Info, Val).Visit(E)) return false; if (!Val.isFixedPoint()) return false; Result = Val.getFixedPoint(); return true; } return false; } static bool EvaluateFixedPointOrInteger(const Expr *E, APFixedPoint &Result, EvalInfo &Info) { if (E->getType()->isIntegerType()) { auto FXSema = Info.Ctx.getFixedPointSemantics(E->getType()); APSInt Val; if (!EvaluateInteger(E, Val, Info)) return false; Result = APFixedPoint(Val, FXSema); return true; } else if (E->getType()->isFixedPointType()) { return EvaluateFixedPoint(E, Result, Info); } return false; } /// Check whether the given declaration can be directly converted to an integral /// rvalue. If not, no diagnostic is produced; there are other things we can /// try. bool IntExprEvaluator::CheckReferencedDecl(const Expr* E, const Decl* D) { // Enums are integer constant exprs. if (const EnumConstantDecl *ECD = dyn_cast(D)) { // Check for signedness/width mismatches between E type and ECD value. bool SameSign = (ECD->getInitVal().isSigned() == E->getType()->isSignedIntegerOrEnumerationType()); bool SameWidth = (ECD->getInitVal().getBitWidth() == Info.Ctx.getIntWidth(E->getType())); if (SameSign && SameWidth) return Success(ECD->getInitVal(), E); else { // Get rid of mismatch (otherwise Success assertions will fail) // by computing a new value matching the type of E. llvm::APSInt Val = ECD->getInitVal(); if (!SameSign) Val.setIsSigned(!ECD->getInitVal().isSigned()); if (!SameWidth) Val = Val.extOrTrunc(Info.Ctx.getIntWidth(E->getType())); return Success(Val, E); } } return false; } /// Values returned by __builtin_classify_type, chosen to match the values /// produced by GCC's builtin. enum class GCCTypeClass { None = -1, Void = 0, Integer = 1, // GCC reserves 2 for character types, but instead classifies them as // integers. Enum = 3, Bool = 4, Pointer = 5, // GCC reserves 6 for references, but appears to never use it (because // expressions never have reference type, presumably). PointerToDataMember = 7, RealFloat = 8, Complex = 9, // GCC reserves 10 for functions, but does not use it since GCC version 6 due // to decay to pointer. (Prior to version 6 it was only used in C++ mode). // GCC claims to reserve 11 for pointers to member functions, but *actually* // uses 12 for that purpose, same as for a class or struct. Maybe it // internally implements a pointer to member as a struct? Who knows. PointerToMemberFunction = 12, // Not a bug, see above. ClassOrStruct = 12, Union = 13, // GCC reserves 14 for arrays, but does not use it since GCC version 6 due to // decay to pointer. (Prior to version 6 it was only used in C++ mode). // GCC reserves 15 for strings, but actually uses 5 (pointer) for string // literals. }; /// EvaluateBuiltinClassifyType - Evaluate __builtin_classify_type the same way /// as GCC. static GCCTypeClass EvaluateBuiltinClassifyType(QualType T, const LangOptions &LangOpts) { assert(!T->isDependentType() && "unexpected dependent type"); QualType CanTy = T.getCanonicalType(); const BuiltinType *BT = dyn_cast(CanTy); switch (CanTy->getTypeClass()) { #define TYPE(ID, BASE) #define DEPENDENT_TYPE(ID, BASE) case Type::ID: #define NON_CANONICAL_TYPE(ID, BASE) case Type::ID: #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(ID, BASE) case Type::ID: #include "clang/AST/TypeNodes.def" case Type::Auto: case Type::DeducedTemplateSpecialization: llvm_unreachable("unexpected non-canonical or dependent type"); case Type::Builtin: switch (BT->getKind()) { #define BUILTIN_TYPE(ID, SINGLETON_ID) #define SIGNED_TYPE(ID, SINGLETON_ID) \ case BuiltinType::ID: return GCCTypeClass::Integer; #define FLOATING_TYPE(ID, SINGLETON_ID) \ case BuiltinType::ID: return GCCTypeClass::RealFloat; #define PLACEHOLDER_TYPE(ID, SINGLETON_ID) \ case BuiltinType::ID: break; #include "clang/AST/BuiltinTypes.def" case BuiltinType::Void: return GCCTypeClass::Void; case BuiltinType::Bool: return GCCTypeClass::Bool; case BuiltinType::Char_U: case BuiltinType::UChar: case BuiltinType::WChar_U: case BuiltinType::Char8: case BuiltinType::Char16: case BuiltinType::Char32: case BuiltinType::UShort: case BuiltinType::UInt: case BuiltinType::ULong: case BuiltinType::ULongLong: case BuiltinType::UInt128: return GCCTypeClass::Integer; case BuiltinType::UShortAccum: case BuiltinType::UAccum: case BuiltinType::ULongAccum: case BuiltinType::UShortFract: case BuiltinType::UFract: case BuiltinType::ULongFract: case BuiltinType::SatUShortAccum: case BuiltinType::SatUAccum: case BuiltinType::SatULongAccum: case BuiltinType::SatUShortFract: case BuiltinType::SatUFract: case BuiltinType::SatULongFract: return GCCTypeClass::None; case BuiltinType::NullPtr: case BuiltinType::ObjCId: case BuiltinType::ObjCClass: case BuiltinType::ObjCSel: #define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \ case BuiltinType::Id: #include "clang/Basic/OpenCLImageTypes.def" #define EXT_OPAQUE_TYPE(ExtType, Id, Ext) \ case BuiltinType::Id: #include "clang/Basic/OpenCLExtensionTypes.def" case BuiltinType::OCLSampler: case BuiltinType::OCLEvent: case BuiltinType::OCLClkEvent: case BuiltinType::OCLQueue: case BuiltinType::OCLReserveID: return GCCTypeClass::None; case BuiltinType::Dependent: llvm_unreachable("unexpected dependent type"); }; llvm_unreachable("unexpected placeholder type"); case Type::Enum: return LangOpts.CPlusPlus ? GCCTypeClass::Enum : GCCTypeClass::Integer; case Type::Pointer: case Type::ConstantArray: case Type::VariableArray: case Type::IncompleteArray: case Type::FunctionNoProto: case Type::FunctionProto: return GCCTypeClass::Pointer; case Type::MemberPointer: return CanTy->isMemberDataPointerType() ? GCCTypeClass::PointerToDataMember : GCCTypeClass::PointerToMemberFunction; case Type::Complex: return GCCTypeClass::Complex; case Type::Record: return CanTy->isUnionType() ? GCCTypeClass::Union : GCCTypeClass::ClassOrStruct; case Type::Atomic: // GCC classifies _Atomic T the same as T. return EvaluateBuiltinClassifyType( CanTy->castAs()->getValueType(), LangOpts); case Type::BlockPointer: case Type::Vector: case Type::ExtVector: case Type::ObjCObject: case Type::ObjCInterface: case Type::ObjCObjectPointer: case Type::Pipe: // GCC classifies vectors as None. We follow its lead and classify all // other types that don't fit into the regular classification the same way. return GCCTypeClass::None; case Type::LValueReference: case Type::RValueReference: llvm_unreachable("invalid type for expression"); } llvm_unreachable("unexpected type class"); } /// EvaluateBuiltinClassifyType - Evaluate __builtin_classify_type the same way /// as GCC. static GCCTypeClass EvaluateBuiltinClassifyType(const CallExpr *E, const LangOptions &LangOpts) { // If no argument was supplied, default to None. This isn't // ideal, however it is what gcc does. if (E->getNumArgs() == 0) return GCCTypeClass::None; // FIXME: Bizarrely, GCC treats a call with more than one argument as not // being an ICE, but still folds it to a constant using the type of the first // argument. return EvaluateBuiltinClassifyType(E->getArg(0)->getType(), LangOpts); } /// EvaluateBuiltinConstantPForLValue - Determine the result of /// __builtin_constant_p when applied to the given pointer. /// /// A pointer is only "constant" if it is null (or a pointer cast to integer) /// or it points to the first character of a string literal. static bool EvaluateBuiltinConstantPForLValue(const APValue &LV) { APValue::LValueBase Base = LV.getLValueBase(); if (Base.isNull()) { // A null base is acceptable. return true; } else if (const Expr *E = Base.dyn_cast()) { if (!isa(E)) return false; return LV.getLValueOffset().isZero(); } else if (Base.is()) { // Surprisingly, GCC considers __builtin_constant_p(&typeid(int)) to // evaluate to true. return true; } else { // Any other base is not constant enough for GCC. return false; } } /// EvaluateBuiltinConstantP - Evaluate __builtin_constant_p as similarly to /// GCC as we can manage. static bool EvaluateBuiltinConstantP(EvalInfo &Info, const Expr *Arg) { // This evaluation is not permitted to have side-effects, so evaluate it in // a speculative evaluation context. SpeculativeEvaluationRAII SpeculativeEval(Info); // Constant-folding is always enabled for the operand of __builtin_constant_p // (even when the enclosing evaluation context otherwise requires a strict // language-specific constant expression). FoldConstant Fold(Info, true); QualType ArgType = Arg->getType(); // __builtin_constant_p always has one operand. The rules which gcc follows // are not precisely documented, but are as follows: // // - If the operand is of integral, floating, complex or enumeration type, // and can be folded to a known value of that type, it returns 1. // - If the operand can be folded to a pointer to the first character // of a string literal (or such a pointer cast to an integral type) // or to a null pointer or an integer cast to a pointer, it returns 1. // // Otherwise, it returns 0. // // FIXME: GCC also intends to return 1 for literals of aggregate types, but // its support for this did not work prior to GCC 9 and is not yet well // understood. if (ArgType->isIntegralOrEnumerationType() || ArgType->isFloatingType() || ArgType->isAnyComplexType() || ArgType->isPointerType() || ArgType->isNullPtrType()) { APValue V; if (!::EvaluateAsRValue(Info, Arg, V)) { Fold.keepDiagnostics(); return false; } // For a pointer (possibly cast to integer), there are special rules. if (V.getKind() == APValue::LValue) return EvaluateBuiltinConstantPForLValue(V); // Otherwise, any constant value is good enough. return V.hasValue(); } // Anything else isn't considered to be sufficiently constant. return false; } /// Retrieves the "underlying object type" of the given expression, /// as used by __builtin_object_size. static QualType getObjectType(APValue::LValueBase B) { if (const ValueDecl *D = B.dyn_cast()) { if (const VarDecl *VD = dyn_cast(D)) return VD->getType(); } else if (const Expr *E = B.get()) { if (isa(E)) return E->getType(); } else if (B.is()) { return B.getTypeInfoType(); } return QualType(); } /// A more selective version of E->IgnoreParenCasts for /// tryEvaluateBuiltinObjectSize. This ignores some casts/parens that serve only /// to change the type of E. /// Ex. For E = `(short*)((char*)(&foo))`, returns `&foo` /// /// Always returns an RValue with a pointer representation. static const Expr *ignorePointerCastsAndParens(const Expr *E) { assert(E->isRValue() && E->getType()->hasPointerRepresentation()); auto *NoParens = E->IgnoreParens(); auto *Cast = dyn_cast(NoParens); if (Cast == nullptr) return NoParens; // We only conservatively allow a few kinds of casts, because this code is // inherently a simple solution that seeks to support the common case. auto CastKind = Cast->getCastKind(); if (CastKind != CK_NoOp && CastKind != CK_BitCast && CastKind != CK_AddressSpaceConversion) return NoParens; auto *SubExpr = Cast->getSubExpr(); if (!SubExpr->getType()->hasPointerRepresentation() || !SubExpr->isRValue()) return NoParens; return ignorePointerCastsAndParens(SubExpr); } /// Checks to see if the given LValue's Designator is at the end of the LValue's /// record layout. e.g. /// struct { struct { int a, b; } fst, snd; } obj; /// obj.fst // no /// obj.snd // yes /// obj.fst.a // no /// obj.fst.b // no /// obj.snd.a // no /// obj.snd.b // yes /// /// Please note: this function is specialized for how __builtin_object_size /// views "objects". /// /// If this encounters an invalid RecordDecl or otherwise cannot determine the /// correct result, it will always return true. static bool isDesignatorAtObjectEnd(const ASTContext &Ctx, const LValue &LVal) { assert(!LVal.Designator.Invalid); auto IsLastOrInvalidFieldDecl = [&Ctx](const FieldDecl *FD, bool &Invalid) { const RecordDecl *Parent = FD->getParent(); Invalid = Parent->isInvalidDecl(); if (Invalid || Parent->isUnion()) return true; const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(Parent); return FD->getFieldIndex() + 1 == Layout.getFieldCount(); }; auto &Base = LVal.getLValueBase(); if (auto *ME = dyn_cast_or_null(Base.dyn_cast())) { if (auto *FD = dyn_cast(ME->getMemberDecl())) { bool Invalid; if (!IsLastOrInvalidFieldDecl(FD, Invalid)) return Invalid; } else if (auto *IFD = dyn_cast(ME->getMemberDecl())) { for (auto *FD : IFD->chain()) { bool Invalid; if (!IsLastOrInvalidFieldDecl(cast(FD), Invalid)) return Invalid; } } } unsigned I = 0; QualType BaseType = getType(Base); if (LVal.Designator.FirstEntryIsAnUnsizedArray) { // If we don't know the array bound, conservatively assume we're looking at // the final array element. ++I; if (BaseType->isIncompleteArrayType()) BaseType = Ctx.getAsArrayType(BaseType)->getElementType(); else BaseType = BaseType->castAs()->getPointeeType(); } for (unsigned E = LVal.Designator.Entries.size(); I != E; ++I) { const auto &Entry = LVal.Designator.Entries[I]; if (BaseType->isArrayType()) { // Because __builtin_object_size treats arrays as objects, we can ignore // the index iff this is the last array in the Designator. if (I + 1 == E) return true; const auto *CAT = cast(Ctx.getAsArrayType(BaseType)); uint64_t Index = Entry.getAsArrayIndex(); if (Index + 1 != CAT->getSize()) return false; BaseType = CAT->getElementType(); } else if (BaseType->isAnyComplexType()) { const auto *CT = BaseType->castAs(); uint64_t Index = Entry.getAsArrayIndex(); if (Index != 1) return false; BaseType = CT->getElementType(); } else if (auto *FD = getAsField(Entry)) { bool Invalid; if (!IsLastOrInvalidFieldDecl(FD, Invalid)) return Invalid; BaseType = FD->getType(); } else { assert(getAsBaseClass(Entry) && "Expecting cast to a base class"); return false; } } return true; } /// Tests to see if the LValue has a user-specified designator (that isn't /// necessarily valid). Note that this always returns 'true' if the LValue has /// an unsized array as its first designator entry, because there's currently no /// way to tell if the user typed *foo or foo[0]. static bool refersToCompleteObject(const LValue &LVal) { if (LVal.Designator.Invalid) return false; if (!LVal.Designator.Entries.empty()) return LVal.Designator.isMostDerivedAnUnsizedArray(); if (!LVal.InvalidBase) return true; // If `E` is a MemberExpr, then the first part of the designator is hiding in // the LValueBase. const auto *E = LVal.Base.dyn_cast(); return !E || !isa(E); } /// Attempts to detect a user writing into a piece of memory that's impossible /// to figure out the size of by just using types. static bool isUserWritingOffTheEnd(const ASTContext &Ctx, const LValue &LVal) { const SubobjectDesignator &Designator = LVal.Designator; // Notes: // - Users can only write off of the end when we have an invalid base. Invalid // bases imply we don't know where the memory came from. // - We used to be a bit more aggressive here; we'd only be conservative if // the array at the end was flexible, or if it had 0 or 1 elements. This // broke some common standard library extensions (PR30346), but was // otherwise seemingly fine. It may be useful to reintroduce this behavior // with some sort of whitelist. OTOH, it seems that GCC is always // conservative with the last element in structs (if it's an array), so our // current behavior is more compatible than a whitelisting approach would // be. return LVal.InvalidBase && Designator.Entries.size() == Designator.MostDerivedPathLength && Designator.MostDerivedIsArrayElement && isDesignatorAtObjectEnd(Ctx, LVal); } /// Converts the given APInt to CharUnits, assuming the APInt is unsigned. /// Fails if the conversion would cause loss of precision. static bool convertUnsignedAPIntToCharUnits(const llvm::APInt &Int, CharUnits &Result) { auto CharUnitsMax = std::numeric_limits::max(); if (Int.ugt(CharUnitsMax)) return false; Result = CharUnits::fromQuantity(Int.getZExtValue()); return true; } /// Helper for tryEvaluateBuiltinObjectSize -- Given an LValue, this will /// determine how many bytes exist from the beginning of the object to either /// the end of the current subobject, or the end of the object itself, depending /// on what the LValue looks like + the value of Type. /// /// If this returns false, the value of Result is undefined. static bool determineEndOffset(EvalInfo &Info, SourceLocation ExprLoc, unsigned Type, const LValue &LVal, CharUnits &EndOffset) { bool DetermineForCompleteObject = refersToCompleteObject(LVal); auto CheckedHandleSizeof = [&](QualType Ty, CharUnits &Result) { if (Ty.isNull() || Ty->isIncompleteType() || Ty->isFunctionType()) return false; return HandleSizeof(Info, ExprLoc, Ty, Result); }; // We want to evaluate the size of the entire object. This is a valid fallback // for when Type=1 and the designator is invalid, because we're asked for an // upper-bound. if (!(Type & 1) || LVal.Designator.Invalid || DetermineForCompleteObject) { // Type=3 wants a lower bound, so we can't fall back to this. if (Type == 3 && !DetermineForCompleteObject) return false; llvm::APInt APEndOffset; if (isBaseAnAllocSizeCall(LVal.getLValueBase()) && getBytesReturnedByAllocSizeCall(Info.Ctx, LVal, APEndOffset)) return convertUnsignedAPIntToCharUnits(APEndOffset, EndOffset); if (LVal.InvalidBase) return false; QualType BaseTy = getObjectType(LVal.getLValueBase()); return CheckedHandleSizeof(BaseTy, EndOffset); } // We want to evaluate the size of a subobject. const SubobjectDesignator &Designator = LVal.Designator; // The following is a moderately common idiom in C: // // struct Foo { int a; char c[1]; }; // struct Foo *F = (struct Foo *)malloc(sizeof(struct Foo) + strlen(Bar)); // strcpy(&F->c[0], Bar); // // In order to not break too much legacy code, we need to support it. if (isUserWritingOffTheEnd(Info.Ctx, LVal)) { // If we can resolve this to an alloc_size call, we can hand that back, // because we know for certain how many bytes there are to write to. llvm::APInt APEndOffset; if (isBaseAnAllocSizeCall(LVal.getLValueBase()) && getBytesReturnedByAllocSizeCall(Info.Ctx, LVal, APEndOffset)) return convertUnsignedAPIntToCharUnits(APEndOffset, EndOffset); // If we cannot determine the size of the initial allocation, then we can't // given an accurate upper-bound. However, we are still able to give // conservative lower-bounds for Type=3. if (Type == 1) return false; } CharUnits BytesPerElem; if (!CheckedHandleSizeof(Designator.MostDerivedType, BytesPerElem)) return false; // According to the GCC documentation, we want the size of the subobject // denoted by the pointer. But that's not quite right -- what we actually // want is the size of the immediately-enclosing array, if there is one. int64_t ElemsRemaining; if (Designator.MostDerivedIsArrayElement && Designator.Entries.size() == Designator.MostDerivedPathLength) { uint64_t ArraySize = Designator.getMostDerivedArraySize(); uint64_t ArrayIndex = Designator.Entries.back().getAsArrayIndex(); ElemsRemaining = ArraySize <= ArrayIndex ? 0 : ArraySize - ArrayIndex; } else { ElemsRemaining = Designator.isOnePastTheEnd() ? 0 : 1; } EndOffset = LVal.getLValueOffset() + BytesPerElem * ElemsRemaining; return true; } /// Tries to evaluate the __builtin_object_size for @p E. If successful, /// returns true and stores the result in @p Size. /// /// If @p WasError is non-null, this will report whether the failure to evaluate /// is to be treated as an Error in IntExprEvaluator. static bool tryEvaluateBuiltinObjectSize(const Expr *E, unsigned Type, EvalInfo &Info, uint64_t &Size) { // Determine the denoted object. LValue LVal; { // The operand of __builtin_object_size is never evaluated for side-effects. // If there are any, but we can determine the pointed-to object anyway, then // ignore the side-effects. SpeculativeEvaluationRAII SpeculativeEval(Info); IgnoreSideEffectsRAII Fold(Info); if (E->isGLValue()) { // It's possible for us to be given GLValues if we're called via // Expr::tryEvaluateObjectSize. APValue RVal; if (!EvaluateAsRValue(Info, E, RVal)) return false; LVal.setFrom(Info.Ctx, RVal); } else if (!EvaluatePointer(ignorePointerCastsAndParens(E), LVal, Info, /*InvalidBaseOK=*/true)) return false; } // If we point to before the start of the object, there are no accessible // bytes. if (LVal.getLValueOffset().isNegative()) { Size = 0; return true; } CharUnits EndOffset; if (!determineEndOffset(Info, E->getExprLoc(), Type, LVal, EndOffset)) return false; // If we've fallen outside of the end offset, just pretend there's nothing to // write to/read from. if (EndOffset <= LVal.getLValueOffset()) Size = 0; else Size = (EndOffset - LVal.getLValueOffset()).getQuantity(); return true; } bool IntExprEvaluator::VisitConstantExpr(const ConstantExpr *E) { llvm::SaveAndRestore InConstantContext(Info.InConstantContext, true); if (E->getResultAPValueKind() != APValue::None) return Success(E->getAPValueResult(), E); return ExprEvaluatorBaseTy::VisitConstantExpr(E); } bool IntExprEvaluator::VisitCallExpr(const CallExpr *E) { if (unsigned BuiltinOp = E->getBuiltinCallee()) return VisitBuiltinCallExpr(E, BuiltinOp); return ExprEvaluatorBaseTy::VisitCallExpr(E); } bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E, unsigned BuiltinOp) { switch (unsigned BuiltinOp = E->getBuiltinCallee()) { default: return ExprEvaluatorBaseTy::VisitCallExpr(E); case Builtin::BI__builtin_dynamic_object_size: case Builtin::BI__builtin_object_size: { // The type was checked when we built the expression. unsigned Type = E->getArg(1)->EvaluateKnownConstInt(Info.Ctx).getZExtValue(); assert(Type <= 3 && "unexpected type"); uint64_t Size; if (tryEvaluateBuiltinObjectSize(E->getArg(0), Type, Info, Size)) return Success(Size, E); if (E->getArg(0)->HasSideEffects(Info.Ctx)) return Success((Type & 2) ? 0 : -1, E); // Expression had no side effects, but we couldn't statically determine the // size of the referenced object. switch (Info.EvalMode) { case EvalInfo::EM_ConstantExpression: case EvalInfo::EM_ConstantFold: case EvalInfo::EM_IgnoreSideEffects: // Leave it to IR generation. return Error(E); case EvalInfo::EM_ConstantExpressionUnevaluated: // Reduce it to a constant now. return Success((Type & 2) ? 0 : -1, E); } llvm_unreachable("unexpected EvalMode"); } case Builtin::BI__builtin_os_log_format_buffer_size: { analyze_os_log::OSLogBufferLayout Layout; analyze_os_log::computeOSLogBufferLayout(Info.Ctx, E, Layout); return Success(Layout.size().getQuantity(), E); } case Builtin::BI__builtin_bswap16: case Builtin::BI__builtin_bswap32: case Builtin::BI__builtin_bswap64: { APSInt Val; if (!EvaluateInteger(E->getArg(0), Val, Info)) return false; return Success(Val.byteSwap(), E); } case Builtin::BI__builtin_classify_type: return Success((int)EvaluateBuiltinClassifyType(E, Info.getLangOpts()), E); case Builtin::BI__builtin_clrsb: case Builtin::BI__builtin_clrsbl: case Builtin::BI__builtin_clrsbll: { APSInt Val; if (!EvaluateInteger(E->getArg(0), Val, Info)) return false; return Success(Val.getBitWidth() - Val.getMinSignedBits(), E); } case Builtin::BI__builtin_clz: case Builtin::BI__builtin_clzl: case Builtin::BI__builtin_clzll: case Builtin::BI__builtin_clzs: { APSInt Val; if (!EvaluateInteger(E->getArg(0), Val, Info)) return false; if (!Val) return Error(E); return Success(Val.countLeadingZeros(), E); } case Builtin::BI__builtin_constant_p: { const Expr *Arg = E->getArg(0); if (EvaluateBuiltinConstantP(Info, Arg)) return Success(true, E); if (Info.InConstantContext || Arg->HasSideEffects(Info.Ctx)) { // Outside a constant context, eagerly evaluate to false in the presence // of side-effects in order to avoid -Wunsequenced false-positives in // a branch on __builtin_constant_p(expr). return Success(false, E); } Info.FFDiag(E, diag::note_invalid_subexpr_in_const_expr); return false; } case Builtin::BI__builtin_is_constant_evaluated: return Success(Info.InConstantContext, E); case Builtin::BI__builtin_ctz: case Builtin::BI__builtin_ctzl: case Builtin::BI__builtin_ctzll: case Builtin::BI__builtin_ctzs: { APSInt Val; if (!EvaluateInteger(E->getArg(0), Val, Info)) return false; if (!Val) return Error(E); return Success(Val.countTrailingZeros(), E); } case Builtin::BI__builtin_eh_return_data_regno: { int Operand = E->getArg(0)->EvaluateKnownConstInt(Info.Ctx).getZExtValue(); Operand = Info.Ctx.getTargetInfo().getEHDataRegisterNumber(Operand); return Success(Operand, E); } case Builtin::BI__builtin_expect: return Visit(E->getArg(0)); case Builtin::BI__builtin_ffs: case Builtin::BI__builtin_ffsl: case Builtin::BI__builtin_ffsll: { APSInt Val; if (!EvaluateInteger(E->getArg(0), Val, Info)) return false; unsigned N = Val.countTrailingZeros(); return Success(N == Val.getBitWidth() ? 0 : N + 1, E); } case Builtin::BI__builtin_fpclassify: { APFloat Val(0.0); if (!EvaluateFloat(E->getArg(5), Val, Info)) return false; unsigned Arg; switch (Val.getCategory()) { case APFloat::fcNaN: Arg = 0; break; case APFloat::fcInfinity: Arg = 1; break; case APFloat::fcNormal: Arg = Val.isDenormal() ? 3 : 2; break; case APFloat::fcZero: Arg = 4; break; } return Visit(E->getArg(Arg)); } case Builtin::BI__builtin_isinf_sign: { APFloat Val(0.0); return EvaluateFloat(E->getArg(0), Val, Info) && Success(Val.isInfinity() ? (Val.isNegative() ? -1 : 1) : 0, E); } case Builtin::BI__builtin_isinf: { APFloat Val(0.0); return EvaluateFloat(E->getArg(0), Val, Info) && Success(Val.isInfinity() ? 1 : 0, E); } case Builtin::BI__builtin_isfinite: { APFloat Val(0.0); return EvaluateFloat(E->getArg(0), Val, Info) && Success(Val.isFinite() ? 1 : 0, E); } case Builtin::BI__builtin_isnan: { APFloat Val(0.0); return EvaluateFloat(E->getArg(0), Val, Info) && Success(Val.isNaN() ? 1 : 0, E); } case Builtin::BI__builtin_isnormal: { APFloat Val(0.0); return EvaluateFloat(E->getArg(0), Val, Info) && Success(Val.isNormal() ? 1 : 0, E); } case Builtin::BI__builtin_parity: case Builtin::BI__builtin_parityl: case Builtin::BI__builtin_parityll: { APSInt Val; if (!EvaluateInteger(E->getArg(0), Val, Info)) return false; return Success(Val.countPopulation() % 2, E); } case Builtin::BI__builtin_popcount: case Builtin::BI__builtin_popcountl: case Builtin::BI__builtin_popcountll: { APSInt Val; if (!EvaluateInteger(E->getArg(0), Val, Info)) return false; return Success(Val.countPopulation(), E); } case Builtin::BIstrlen: case Builtin::BIwcslen: // A call to strlen is not a constant expression. if (Info.getLangOpts().CPlusPlus11) Info.CCEDiag(E, diag::note_constexpr_invalid_function) << /*isConstexpr*/0 << /*isConstructor*/0 << (std::string("'") + Info.Ctx.BuiltinInfo.getName(BuiltinOp) + "'"); else Info.CCEDiag(E, diag::note_invalid_subexpr_in_const_expr); LLVM_FALLTHROUGH; case Builtin::BI__builtin_strlen: case Builtin::BI__builtin_wcslen: { // As an extension, we support __builtin_strlen() as a constant expression, // and support folding strlen() to a constant. LValue String; if (!EvaluatePointer(E->getArg(0), String, Info)) return false; QualType CharTy = E->getArg(0)->getType()->getPointeeType(); // Fast path: if it's a string literal, search the string value. if (const StringLiteral *S = dyn_cast_or_null( String.getLValueBase().dyn_cast())) { // The string literal may have embedded null characters. Find the first // one and truncate there. StringRef Str = S->getBytes(); int64_t Off = String.Offset.getQuantity(); if (Off >= 0 && (uint64_t)Off <= (uint64_t)Str.size() && S->getCharByteWidth() == 1 && // FIXME: Add fast-path for wchar_t too. Info.Ctx.hasSameUnqualifiedType(CharTy, Info.Ctx.CharTy)) { Str = Str.substr(Off); StringRef::size_type Pos = Str.find(0); if (Pos != StringRef::npos) Str = Str.substr(0, Pos); return Success(Str.size(), E); } // Fall through to slow path to issue appropriate diagnostic. } // Slow path: scan the bytes of the string looking for the terminating 0. for (uint64_t Strlen = 0; /**/; ++Strlen) { APValue Char; if (!handleLValueToRValueConversion(Info, E, CharTy, String, Char) || !Char.isInt()) return false; if (!Char.getInt()) return Success(Strlen, E); if (!HandleLValueArrayAdjustment(Info, E, String, CharTy, 1)) return false; } } case Builtin::BIstrcmp: case Builtin::BIwcscmp: case Builtin::BIstrncmp: case Builtin::BIwcsncmp: case Builtin::BImemcmp: case Builtin::BIbcmp: case Builtin::BIwmemcmp: // A call to strlen is not a constant expression. if (Info.getLangOpts().CPlusPlus11) Info.CCEDiag(E, diag::note_constexpr_invalid_function) << /*isConstexpr*/0 << /*isConstructor*/0 << (std::string("'") + Info.Ctx.BuiltinInfo.getName(BuiltinOp) + "'"); else Info.CCEDiag(E, diag::note_invalid_subexpr_in_const_expr); LLVM_FALLTHROUGH; case Builtin::BI__builtin_strcmp: case Builtin::BI__builtin_wcscmp: case Builtin::BI__builtin_strncmp: case Builtin::BI__builtin_wcsncmp: case Builtin::BI__builtin_memcmp: case Builtin::BI__builtin_bcmp: case Builtin::BI__builtin_wmemcmp: { LValue String1, String2; if (!EvaluatePointer(E->getArg(0), String1, Info) || !EvaluatePointer(E->getArg(1), String2, Info)) return false; uint64_t MaxLength = uint64_t(-1); if (BuiltinOp != Builtin::BIstrcmp && BuiltinOp != Builtin::BIwcscmp && BuiltinOp != Builtin::BI__builtin_strcmp && BuiltinOp != Builtin::BI__builtin_wcscmp) { APSInt N; if (!EvaluateInteger(E->getArg(2), N, Info)) return false; MaxLength = N.getExtValue(); } // Empty substrings compare equal by definition. if (MaxLength == 0u) return Success(0, E); if (!String1.checkNullPointerForFoldAccess(Info, E, AK_Read) || !String2.checkNullPointerForFoldAccess(Info, E, AK_Read) || String1.Designator.Invalid || String2.Designator.Invalid) return false; QualType CharTy1 = String1.Designator.getType(Info.Ctx); QualType CharTy2 = String2.Designator.getType(Info.Ctx); bool IsRawByte = BuiltinOp == Builtin::BImemcmp || BuiltinOp == Builtin::BIbcmp || BuiltinOp == Builtin::BI__builtin_memcmp || BuiltinOp == Builtin::BI__builtin_bcmp; assert(IsRawByte || (Info.Ctx.hasSameUnqualifiedType( CharTy1, E->getArg(0)->getType()->getPointeeType()) && Info.Ctx.hasSameUnqualifiedType(CharTy1, CharTy2))); const auto &ReadCurElems = [&](APValue &Char1, APValue &Char2) { return handleLValueToRValueConversion(Info, E, CharTy1, String1, Char1) && handleLValueToRValueConversion(Info, E, CharTy2, String2, Char2) && Char1.isInt() && Char2.isInt(); }; const auto &AdvanceElems = [&] { return HandleLValueArrayAdjustment(Info, E, String1, CharTy1, 1) && HandleLValueArrayAdjustment(Info, E, String2, CharTy2, 1); }; if (IsRawByte) { uint64_t BytesRemaining = MaxLength; // Pointers to const void may point to objects of incomplete type. if (CharTy1->isIncompleteType()) { Info.FFDiag(E, diag::note_constexpr_ltor_incomplete_type) << CharTy1; return false; } if (CharTy2->isIncompleteType()) { Info.FFDiag(E, diag::note_constexpr_ltor_incomplete_type) << CharTy2; return false; } uint64_t CharTy1Width{Info.Ctx.getTypeSize(CharTy1)}; CharUnits CharTy1Size = Info.Ctx.toCharUnitsFromBits(CharTy1Width); // Give up on comparing between elements with disparate widths. if (CharTy1Size != Info.Ctx.getTypeSizeInChars(CharTy2)) return false; uint64_t BytesPerElement = CharTy1Size.getQuantity(); assert(BytesRemaining && "BytesRemaining should not be zero: the " "following loop considers at least one element"); while (true) { APValue Char1, Char2; if (!ReadCurElems(Char1, Char2)) return false; // We have compatible in-memory widths, but a possible type and // (for `bool`) internal representation mismatch. // Assuming two's complement representation, including 0 for `false` and // 1 for `true`, we can check an appropriate number of elements for // equality even if they are not byte-sized. APSInt Char1InMem = Char1.getInt().extOrTrunc(CharTy1Width); APSInt Char2InMem = Char2.getInt().extOrTrunc(CharTy1Width); if (Char1InMem.ne(Char2InMem)) { // If the elements are byte-sized, then we can produce a three-way // comparison result in a straightforward manner. if (BytesPerElement == 1u) { // memcmp always compares unsigned chars. return Success(Char1InMem.ult(Char2InMem) ? -1 : 1, E); } // The result is byte-order sensitive, and we have multibyte elements. // FIXME: We can compare the remaining bytes in the correct order. return false; } if (!AdvanceElems()) return false; if (BytesRemaining <= BytesPerElement) break; BytesRemaining -= BytesPerElement; } // Enough elements are equal to account for the memcmp limit. return Success(0, E); } bool StopAtNull = (BuiltinOp != Builtin::BImemcmp && BuiltinOp != Builtin::BIbcmp && BuiltinOp != Builtin::BIwmemcmp && BuiltinOp != Builtin::BI__builtin_memcmp && BuiltinOp != Builtin::BI__builtin_bcmp && BuiltinOp != Builtin::BI__builtin_wmemcmp); bool IsWide = BuiltinOp == Builtin::BIwcscmp || BuiltinOp == Builtin::BIwcsncmp || BuiltinOp == Builtin::BIwmemcmp || BuiltinOp == Builtin::BI__builtin_wcscmp || BuiltinOp == Builtin::BI__builtin_wcsncmp || BuiltinOp == Builtin::BI__builtin_wmemcmp; for (; MaxLength; --MaxLength) { APValue Char1, Char2; if (!ReadCurElems(Char1, Char2)) return false; if (Char1.getInt() != Char2.getInt()) { if (IsWide) // wmemcmp compares with wchar_t signedness. return Success(Char1.getInt() < Char2.getInt() ? -1 : 1, E); // memcmp always compares unsigned chars. return Success(Char1.getInt().ult(Char2.getInt()) ? -1 : 1, E); } if (StopAtNull && !Char1.getInt()) return Success(0, E); assert(!(StopAtNull && !Char2.getInt())); if (!AdvanceElems()) return false; } // We hit the strncmp / memcmp limit. return Success(0, E); } case Builtin::BI__atomic_always_lock_free: case Builtin::BI__atomic_is_lock_free: case Builtin::BI__c11_atomic_is_lock_free: { APSInt SizeVal; if (!EvaluateInteger(E->getArg(0), SizeVal, Info)) return false; // For __atomic_is_lock_free(sizeof(_Atomic(T))), if the size is a power // of two less than the maximum inline atomic width, we know it is // lock-free. If the size isn't a power of two, or greater than the // maximum alignment where we promote atomics, we know it is not lock-free // (at least not in the sense of atomic_is_lock_free). Otherwise, // the answer can only be determined at runtime; for example, 16-byte // atomics have lock-free implementations on some, but not all, // x86-64 processors. // Check power-of-two. CharUnits Size = CharUnits::fromQuantity(SizeVal.getZExtValue()); if (Size.isPowerOfTwo()) { // Check against inlining width. unsigned InlineWidthBits = Info.Ctx.getTargetInfo().getMaxAtomicInlineWidth(); if (Size <= Info.Ctx.toCharUnitsFromBits(InlineWidthBits)) { if (BuiltinOp == Builtin::BI__c11_atomic_is_lock_free || Size == CharUnits::One() || E->getArg(1)->isNullPointerConstant(Info.Ctx, Expr::NPC_NeverValueDependent)) // OK, we will inline appropriately-aligned operations of this size, // and _Atomic(T) is appropriately-aligned. return Success(1, E); QualType PointeeType = E->getArg(1)->IgnoreImpCasts()->getType()-> castAs()->getPointeeType(); if (!PointeeType->isIncompleteType() && Info.Ctx.getTypeAlignInChars(PointeeType) >= Size) { // OK, we will inline operations on this object. return Success(1, E); } } } + // Avoid emiting call for runtime decision on PowerPC 32-bit + // The lock free possibilities on this platform are covered by the lines + // above and we know in advance other cases require lock + if (Info.Ctx.getTargetInfo().getTriple().getArch() == llvm::Triple::ppc) { + return Success(0, E); + } + return BuiltinOp == Builtin::BI__atomic_always_lock_free ? Success(0, E) : Error(E); } case Builtin::BIomp_is_initial_device: // We can decide statically which value the runtime would return if called. return Success(Info.getLangOpts().OpenMPIsDevice ? 0 : 1, E); case Builtin::BI__builtin_add_overflow: case Builtin::BI__builtin_sub_overflow: case Builtin::BI__builtin_mul_overflow: case Builtin::BI__builtin_sadd_overflow: case Builtin::BI__builtin_uadd_overflow: case Builtin::BI__builtin_uaddl_overflow: case Builtin::BI__builtin_uaddll_overflow: case Builtin::BI__builtin_usub_overflow: case Builtin::BI__builtin_usubl_overflow: case Builtin::BI__builtin_usubll_overflow: case Builtin::BI__builtin_umul_overflow: case Builtin::BI__builtin_umull_overflow: case Builtin::BI__builtin_umulll_overflow: case Builtin::BI__builtin_saddl_overflow: case Builtin::BI__builtin_saddll_overflow: case Builtin::BI__builtin_ssub_overflow: case Builtin::BI__builtin_ssubl_overflow: case Builtin::BI__builtin_ssubll_overflow: case Builtin::BI__builtin_smul_overflow: case Builtin::BI__builtin_smull_overflow: case Builtin::BI__builtin_smulll_overflow: { LValue ResultLValue; APSInt LHS, RHS; QualType ResultType = E->getArg(2)->getType()->getPointeeType(); if (!EvaluateInteger(E->getArg(0), LHS, Info) || !EvaluateInteger(E->getArg(1), RHS, Info) || !EvaluatePointer(E->getArg(2), ResultLValue, Info)) return false; APSInt Result; bool DidOverflow = false; // If the types don't have to match, enlarge all 3 to the largest of them. if (BuiltinOp == Builtin::BI__builtin_add_overflow || BuiltinOp == Builtin::BI__builtin_sub_overflow || BuiltinOp == Builtin::BI__builtin_mul_overflow) { bool IsSigned = LHS.isSigned() || RHS.isSigned() || ResultType->isSignedIntegerOrEnumerationType(); bool AllSigned = LHS.isSigned() && RHS.isSigned() && ResultType->isSignedIntegerOrEnumerationType(); uint64_t LHSSize = LHS.getBitWidth(); uint64_t RHSSize = RHS.getBitWidth(); uint64_t ResultSize = Info.Ctx.getTypeSize(ResultType); uint64_t MaxBits = std::max(std::max(LHSSize, RHSSize), ResultSize); // Add an additional bit if the signedness isn't uniformly agreed to. We // could do this ONLY if there is a signed and an unsigned that both have // MaxBits, but the code to check that is pretty nasty. The issue will be // caught in the shrink-to-result later anyway. if (IsSigned && !AllSigned) ++MaxBits; LHS = APSInt(LHS.extOrTrunc(MaxBits), !IsSigned); RHS = APSInt(RHS.extOrTrunc(MaxBits), !IsSigned); Result = APSInt(MaxBits, !IsSigned); } // Find largest int. switch (BuiltinOp) { default: llvm_unreachable("Invalid value for BuiltinOp"); case Builtin::BI__builtin_add_overflow: case Builtin::BI__builtin_sadd_overflow: case Builtin::BI__builtin_saddl_overflow: case Builtin::BI__builtin_saddll_overflow: case Builtin::BI__builtin_uadd_overflow: case Builtin::BI__builtin_uaddl_overflow: case Builtin::BI__builtin_uaddll_overflow: Result = LHS.isSigned() ? LHS.sadd_ov(RHS, DidOverflow) : LHS.uadd_ov(RHS, DidOverflow); break; case Builtin::BI__builtin_sub_overflow: case Builtin::BI__builtin_ssub_overflow: case Builtin::BI__builtin_ssubl_overflow: case Builtin::BI__builtin_ssubll_overflow: case Builtin::BI__builtin_usub_overflow: case Builtin::BI__builtin_usubl_overflow: case Builtin::BI__builtin_usubll_overflow: Result = LHS.isSigned() ? LHS.ssub_ov(RHS, DidOverflow) : LHS.usub_ov(RHS, DidOverflow); break; case Builtin::BI__builtin_mul_overflow: case Builtin::BI__builtin_smul_overflow: case Builtin::BI__builtin_smull_overflow: case Builtin::BI__builtin_smulll_overflow: case Builtin::BI__builtin_umul_overflow: case Builtin::BI__builtin_umull_overflow: case Builtin::BI__builtin_umulll_overflow: Result = LHS.isSigned() ? LHS.smul_ov(RHS, DidOverflow) : LHS.umul_ov(RHS, DidOverflow); break; } // In the case where multiple sizes are allowed, truncate and see if // the values are the same. if (BuiltinOp == Builtin::BI__builtin_add_overflow || BuiltinOp == Builtin::BI__builtin_sub_overflow || BuiltinOp == Builtin::BI__builtin_mul_overflow) { // APSInt doesn't have a TruncOrSelf, so we use extOrTrunc instead, // since it will give us the behavior of a TruncOrSelf in the case where // its parameter <= its size. We previously set Result to be at least the // type-size of the result, so getTypeSize(ResultType) <= Result.BitWidth // will work exactly like TruncOrSelf. APSInt Temp = Result.extOrTrunc(Info.Ctx.getTypeSize(ResultType)); Temp.setIsSigned(ResultType->isSignedIntegerOrEnumerationType()); if (!APSInt::isSameValue(Temp, Result)) DidOverflow = true; Result = Temp; } APValue APV{Result}; if (!handleAssignment(Info, E, ResultLValue, ResultType, APV)) return false; return Success(DidOverflow, E); } } } /// Determine whether this is a pointer past the end of the complete /// object referred to by the lvalue. static bool isOnePastTheEndOfCompleteObject(const ASTContext &Ctx, const LValue &LV) { // A null pointer can be viewed as being "past the end" but we don't // choose to look at it that way here. if (!LV.getLValueBase()) return false; // If the designator is valid and refers to a subobject, we're not pointing // past the end. if (!LV.getLValueDesignator().Invalid && !LV.getLValueDesignator().isOnePastTheEnd()) return false; // A pointer to an incomplete type might be past-the-end if the type's size is // zero. We cannot tell because the type is incomplete. QualType Ty = getType(LV.getLValueBase()); if (Ty->isIncompleteType()) return true; // We're a past-the-end pointer if we point to the byte after the object, // no matter what our type or path is. auto Size = Ctx.getTypeSizeInChars(Ty); return LV.getLValueOffset() == Size; } namespace { /// Data recursive integer evaluator of certain binary operators. /// /// We use a data recursive algorithm for binary operators so that we are able /// to handle extreme cases of chained binary operators without causing stack /// overflow. class DataRecursiveIntBinOpEvaluator { struct EvalResult { APValue Val; bool Failed; EvalResult() : Failed(false) { } void swap(EvalResult &RHS) { Val.swap(RHS.Val); Failed = RHS.Failed; RHS.Failed = false; } }; struct Job { const Expr *E; EvalResult LHSResult; // meaningful only for binary operator expression. enum { AnyExprKind, BinOpKind, BinOpVisitedLHSKind } Kind; Job() = default; Job(Job &&) = default; void startSpeculativeEval(EvalInfo &Info) { SpecEvalRAII = SpeculativeEvaluationRAII(Info); } private: SpeculativeEvaluationRAII SpecEvalRAII; }; SmallVector Queue; IntExprEvaluator &IntEval; EvalInfo &Info; APValue &FinalResult; public: DataRecursiveIntBinOpEvaluator(IntExprEvaluator &IntEval, APValue &Result) : IntEval(IntEval), Info(IntEval.getEvalInfo()), FinalResult(Result) { } /// True if \param E is a binary operator that we are going to handle /// data recursively. /// We handle binary operators that are comma, logical, or that have operands /// with integral or enumeration type. static bool shouldEnqueue(const BinaryOperator *E) { return E->getOpcode() == BO_Comma || E->isLogicalOp() || (E->isRValue() && E->getType()->isIntegralOrEnumerationType() && E->getLHS()->getType()->isIntegralOrEnumerationType() && E->getRHS()->getType()->isIntegralOrEnumerationType()); } bool Traverse(const BinaryOperator *E) { enqueue(E); EvalResult PrevResult; while (!Queue.empty()) process(PrevResult); if (PrevResult.Failed) return false; FinalResult.swap(PrevResult.Val); return true; } private: bool Success(uint64_t Value, const Expr *E, APValue &Result) { return IntEval.Success(Value, E, Result); } bool Success(const APSInt &Value, const Expr *E, APValue &Result) { return IntEval.Success(Value, E, Result); } bool Error(const Expr *E) { return IntEval.Error(E); } bool Error(const Expr *E, diag::kind D) { return IntEval.Error(E, D); } OptionalDiagnostic CCEDiag(const Expr *E, diag::kind D) { return Info.CCEDiag(E, D); } // Returns true if visiting the RHS is necessary, false otherwise. bool VisitBinOpLHSOnly(EvalResult &LHSResult, const BinaryOperator *E, bool &SuppressRHSDiags); bool VisitBinOp(const EvalResult &LHSResult, const EvalResult &RHSResult, const BinaryOperator *E, APValue &Result); void EvaluateExpr(const Expr *E, EvalResult &Result) { Result.Failed = !Evaluate(Result.Val, Info, E); if (Result.Failed) Result.Val = APValue(); } void process(EvalResult &Result); void enqueue(const Expr *E) { E = E->IgnoreParens(); Queue.resize(Queue.size()+1); Queue.back().E = E; Queue.back().Kind = Job::AnyExprKind; } }; } bool DataRecursiveIntBinOpEvaluator:: VisitBinOpLHSOnly(EvalResult &LHSResult, const BinaryOperator *E, bool &SuppressRHSDiags) { if (E->getOpcode() == BO_Comma) { // Ignore LHS but note if we could not evaluate it. if (LHSResult.Failed) return Info.noteSideEffect(); return true; } if (E->isLogicalOp()) { bool LHSAsBool; if (!LHSResult.Failed && HandleConversionToBool(LHSResult.Val, LHSAsBool)) { // We were able to evaluate the LHS, see if we can get away with not // evaluating the RHS: 0 && X -> 0, 1 || X -> 1 if (LHSAsBool == (E->getOpcode() == BO_LOr)) { Success(LHSAsBool, E, LHSResult.Val); return false; // Ignore RHS } } else { LHSResult.Failed = true; // Since we weren't able to evaluate the left hand side, it // might have had side effects. if (!Info.noteSideEffect()) return false; // We can't evaluate the LHS; however, sometimes the result // is determined by the RHS: X && 0 -> 0, X || 1 -> 1. // Don't ignore RHS and suppress diagnostics from this arm. SuppressRHSDiags = true; } return true; } assert(E->getLHS()->getType()->isIntegralOrEnumerationType() && E->getRHS()->getType()->isIntegralOrEnumerationType()); if (LHSResult.Failed && !Info.noteFailure()) return false; // Ignore RHS; return true; } static void addOrSubLValueAsInteger(APValue &LVal, const APSInt &Index, bool IsSub) { // Compute the new offset in the appropriate width, wrapping at 64 bits. // FIXME: When compiling for a 32-bit target, we should use 32-bit // offsets. assert(!LVal.hasLValuePath() && "have designator for integer lvalue"); CharUnits &Offset = LVal.getLValueOffset(); uint64_t Offset64 = Offset.getQuantity(); uint64_t Index64 = Index.extOrTrunc(64).getZExtValue(); Offset = CharUnits::fromQuantity(IsSub ? Offset64 - Index64 : Offset64 + Index64); } bool DataRecursiveIntBinOpEvaluator:: VisitBinOp(const EvalResult &LHSResult, const EvalResult &RHSResult, const BinaryOperator *E, APValue &Result) { if (E->getOpcode() == BO_Comma) { if (RHSResult.Failed) return false; Result = RHSResult.Val; return true; } if (E->isLogicalOp()) { bool lhsResult, rhsResult; bool LHSIsOK = HandleConversionToBool(LHSResult.Val, lhsResult); bool RHSIsOK = HandleConversionToBool(RHSResult.Val, rhsResult); if (LHSIsOK) { if (RHSIsOK) { if (E->getOpcode() == BO_LOr) return Success(lhsResult || rhsResult, E, Result); else return Success(lhsResult && rhsResult, E, Result); } } else { if (RHSIsOK) { // We can't evaluate the LHS; however, sometimes the result // is determined by the RHS: X && 0 -> 0, X || 1 -> 1. if (rhsResult == (E->getOpcode() == BO_LOr)) return Success(rhsResult, E, Result); } } return false; } assert(E->getLHS()->getType()->isIntegralOrEnumerationType() && E->getRHS()->getType()->isIntegralOrEnumerationType()); if (LHSResult.Failed || RHSResult.Failed) return false; const APValue &LHSVal = LHSResult.Val; const APValue &RHSVal = RHSResult.Val; // Handle cases like (unsigned long)&a + 4. if (E->isAdditiveOp() && LHSVal.isLValue() && RHSVal.isInt()) { Result = LHSVal; addOrSubLValueAsInteger(Result, RHSVal.getInt(), E->getOpcode() == BO_Sub); return true; } // Handle cases like 4 + (unsigned long)&a if (E->getOpcode() == BO_Add && RHSVal.isLValue() && LHSVal.isInt()) { Result = RHSVal; addOrSubLValueAsInteger(Result, LHSVal.getInt(), /*IsSub*/false); return true; } if (E->getOpcode() == BO_Sub && LHSVal.isLValue() && RHSVal.isLValue()) { // Handle (intptr_t)&&A - (intptr_t)&&B. if (!LHSVal.getLValueOffset().isZero() || !RHSVal.getLValueOffset().isZero()) return false; const Expr *LHSExpr = LHSVal.getLValueBase().dyn_cast(); const Expr *RHSExpr = RHSVal.getLValueBase().dyn_cast(); if (!LHSExpr || !RHSExpr) return false; const AddrLabelExpr *LHSAddrExpr = dyn_cast(LHSExpr); const AddrLabelExpr *RHSAddrExpr = dyn_cast(RHSExpr); if (!LHSAddrExpr || !RHSAddrExpr) return false; // Make sure both labels come from the same function. if (LHSAddrExpr->getLabel()->getDeclContext() != RHSAddrExpr->getLabel()->getDeclContext()) return false; Result = APValue(LHSAddrExpr, RHSAddrExpr); return true; } // All the remaining cases expect both operands to be an integer if (!LHSVal.isInt() || !RHSVal.isInt()) return Error(E); // Set up the width and signedness manually, in case it can't be deduced // from the operation we're performing. // FIXME: Don't do this in the cases where we can deduce it. APSInt Value(Info.Ctx.getIntWidth(E->getType()), E->getType()->isUnsignedIntegerOrEnumerationType()); if (!handleIntIntBinOp(Info, E, LHSVal.getInt(), E->getOpcode(), RHSVal.getInt(), Value)) return false; return Success(Value, E, Result); } void DataRecursiveIntBinOpEvaluator::process(EvalResult &Result) { Job &job = Queue.back(); switch (job.Kind) { case Job::AnyExprKind: { if (const BinaryOperator *Bop = dyn_cast(job.E)) { if (shouldEnqueue(Bop)) { job.Kind = Job::BinOpKind; enqueue(Bop->getLHS()); return; } } EvaluateExpr(job.E, Result); Queue.pop_back(); return; } case Job::BinOpKind: { const BinaryOperator *Bop = cast(job.E); bool SuppressRHSDiags = false; if (!VisitBinOpLHSOnly(Result, Bop, SuppressRHSDiags)) { Queue.pop_back(); return; } if (SuppressRHSDiags) job.startSpeculativeEval(Info); job.LHSResult.swap(Result); job.Kind = Job::BinOpVisitedLHSKind; enqueue(Bop->getRHS()); return; } case Job::BinOpVisitedLHSKind: { const BinaryOperator *Bop = cast(job.E); EvalResult RHS; RHS.swap(Result); Result.Failed = !VisitBinOp(job.LHSResult, RHS, Bop, Result.Val); Queue.pop_back(); return; } } llvm_unreachable("Invalid Job::Kind!"); } namespace { /// Used when we determine that we should fail, but can keep evaluating prior to /// noting that we had a failure. class DelayedNoteFailureRAII { EvalInfo &Info; bool NoteFailure; public: DelayedNoteFailureRAII(EvalInfo &Info, bool NoteFailure = true) : Info(Info), NoteFailure(NoteFailure) {} ~DelayedNoteFailureRAII() { if (NoteFailure) { bool ContinueAfterFailure = Info.noteFailure(); (void)ContinueAfterFailure; assert(ContinueAfterFailure && "Shouldn't have kept evaluating on failure."); } } }; } template static bool EvaluateComparisonBinaryOperator(EvalInfo &Info, const BinaryOperator *E, SuccessCB &&Success, AfterCB &&DoAfter) { assert(E->isComparisonOp() && "expected comparison operator"); assert((E->getOpcode() == BO_Cmp || E->getType()->isIntegralOrEnumerationType()) && "unsupported binary expression evaluation"); auto Error = [&](const Expr *E) { Info.FFDiag(E, diag::note_invalid_subexpr_in_const_expr); return false; }; using CCR = ComparisonCategoryResult; bool IsRelational = E->isRelationalOp(); bool IsEquality = E->isEqualityOp(); if (E->getOpcode() == BO_Cmp) { const ComparisonCategoryInfo &CmpInfo = Info.Ctx.CompCategories.getInfoForType(E->getType()); IsRelational = CmpInfo.isOrdered(); IsEquality = CmpInfo.isEquality(); } QualType LHSTy = E->getLHS()->getType(); QualType RHSTy = E->getRHS()->getType(); if (LHSTy->isIntegralOrEnumerationType() && RHSTy->isIntegralOrEnumerationType()) { APSInt LHS, RHS; bool LHSOK = EvaluateInteger(E->getLHS(), LHS, Info); if (!LHSOK && !Info.noteFailure()) return false; if (!EvaluateInteger(E->getRHS(), RHS, Info) || !LHSOK) return false; if (LHS < RHS) return Success(CCR::Less, E); if (LHS > RHS) return Success(CCR::Greater, E); return Success(CCR::Equal, E); } if (LHSTy->isFixedPointType() || RHSTy->isFixedPointType()) { APFixedPoint LHSFX(Info.Ctx.getFixedPointSemantics(LHSTy)); APFixedPoint RHSFX(Info.Ctx.getFixedPointSemantics(RHSTy)); bool LHSOK = EvaluateFixedPointOrInteger(E->getLHS(), LHSFX, Info); if (!LHSOK && !Info.noteFailure()) return false; if (!EvaluateFixedPointOrInteger(E->getRHS(), RHSFX, Info) || !LHSOK) return false; if (LHSFX < RHSFX) return Success(CCR::Less, E); if (LHSFX > RHSFX) return Success(CCR::Greater, E); return Success(CCR::Equal, E); } if (LHSTy->isAnyComplexType() || RHSTy->isAnyComplexType()) { ComplexValue LHS, RHS; bool LHSOK; if (E->isAssignmentOp()) { LValue LV; EvaluateLValue(E->getLHS(), LV, Info); LHSOK = false; } else if (LHSTy->isRealFloatingType()) { LHSOK = EvaluateFloat(E->getLHS(), LHS.FloatReal, Info); if (LHSOK) { LHS.makeComplexFloat(); LHS.FloatImag = APFloat(LHS.FloatReal.getSemantics()); } } else { LHSOK = EvaluateComplex(E->getLHS(), LHS, Info); } if (!LHSOK && !Info.noteFailure()) return false; if (E->getRHS()->getType()->isRealFloatingType()) { if (!EvaluateFloat(E->getRHS(), RHS.FloatReal, Info) || !LHSOK) return false; RHS.makeComplexFloat(); RHS.FloatImag = APFloat(RHS.FloatReal.getSemantics()); } else if (!EvaluateComplex(E->getRHS(), RHS, Info) || !LHSOK) return false; if (LHS.isComplexFloat()) { APFloat::cmpResult CR_r = LHS.getComplexFloatReal().compare(RHS.getComplexFloatReal()); APFloat::cmpResult CR_i = LHS.getComplexFloatImag().compare(RHS.getComplexFloatImag()); bool IsEqual = CR_r == APFloat::cmpEqual && CR_i == APFloat::cmpEqual; return Success(IsEqual ? CCR::Equal : CCR::Nonequal, E); } else { assert(IsEquality && "invalid complex comparison"); bool IsEqual = LHS.getComplexIntReal() == RHS.getComplexIntReal() && LHS.getComplexIntImag() == RHS.getComplexIntImag(); return Success(IsEqual ? CCR::Equal : CCR::Nonequal, E); } } if (LHSTy->isRealFloatingType() && RHSTy->isRealFloatingType()) { APFloat RHS(0.0), LHS(0.0); bool LHSOK = EvaluateFloat(E->getRHS(), RHS, Info); if (!LHSOK && !Info.noteFailure()) return false; if (!EvaluateFloat(E->getLHS(), LHS, Info) || !LHSOK) return false; assert(E->isComparisonOp() && "Invalid binary operator!"); auto GetCmpRes = [&]() { switch (LHS.compare(RHS)) { case APFloat::cmpEqual: return CCR::Equal; case APFloat::cmpLessThan: return CCR::Less; case APFloat::cmpGreaterThan: return CCR::Greater; case APFloat::cmpUnordered: return CCR::Unordered; } llvm_unreachable("Unrecognised APFloat::cmpResult enum"); }; return Success(GetCmpRes(), E); } if (LHSTy->isPointerType() && RHSTy->isPointerType()) { LValue LHSValue, RHSValue; bool LHSOK = EvaluatePointer(E->getLHS(), LHSValue, Info); if (!LHSOK && !Info.noteFailure()) return false; if (!EvaluatePointer(E->getRHS(), RHSValue, Info) || !LHSOK) return false; // Reject differing bases from the normal codepath; we special-case // comparisons to null. if (!HasSameBase(LHSValue, RHSValue)) { // Inequalities and subtractions between unrelated pointers have // unspecified or undefined behavior. if (!IsEquality) return Error(E); // A constant address may compare equal to the address of a symbol. // The one exception is that address of an object cannot compare equal // to a null pointer constant. if ((!LHSValue.Base && !LHSValue.Offset.isZero()) || (!RHSValue.Base && !RHSValue.Offset.isZero())) return Error(E); // It's implementation-defined whether distinct literals will have // distinct addresses. In clang, the result of such a comparison is // unspecified, so it is not a constant expression. However, we do know // that the address of a literal will be non-null. if ((IsLiteralLValue(LHSValue) || IsLiteralLValue(RHSValue)) && LHSValue.Base && RHSValue.Base) return Error(E); // We can't tell whether weak symbols will end up pointing to the same // object. if (IsWeakLValue(LHSValue) || IsWeakLValue(RHSValue)) return Error(E); // We can't compare the address of the start of one object with the // past-the-end address of another object, per C++ DR1652. if ((LHSValue.Base && LHSValue.Offset.isZero() && isOnePastTheEndOfCompleteObject(Info.Ctx, RHSValue)) || (RHSValue.Base && RHSValue.Offset.isZero() && isOnePastTheEndOfCompleteObject(Info.Ctx, LHSValue))) return Error(E); // We can't tell whether an object is at the same address as another // zero sized object. if ((RHSValue.Base && isZeroSized(LHSValue)) || (LHSValue.Base && isZeroSized(RHSValue))) return Error(E); return Success(CCR::Nonequal, E); } const CharUnits &LHSOffset = LHSValue.getLValueOffset(); const CharUnits &RHSOffset = RHSValue.getLValueOffset(); SubobjectDesignator &LHSDesignator = LHSValue.getLValueDesignator(); SubobjectDesignator &RHSDesignator = RHSValue.getLValueDesignator(); // C++11 [expr.rel]p3: // Pointers to void (after pointer conversions) can be compared, with a // result defined as follows: If both pointers represent the same // address or are both the null pointer value, the result is true if the // operator is <= or >= and false otherwise; otherwise the result is // unspecified. // We interpret this as applying to pointers to *cv* void. if (LHSTy->isVoidPointerType() && LHSOffset != RHSOffset && IsRelational) Info.CCEDiag(E, diag::note_constexpr_void_comparison); // C++11 [expr.rel]p2: // - If two pointers point to non-static data members of the same object, // or to subobjects or array elements fo such members, recursively, the // pointer to the later declared member compares greater provided the // two members have the same access control and provided their class is // not a union. // [...] // - Otherwise pointer comparisons are unspecified. if (!LHSDesignator.Invalid && !RHSDesignator.Invalid && IsRelational) { bool WasArrayIndex; unsigned Mismatch = FindDesignatorMismatch( getType(LHSValue.Base), LHSDesignator, RHSDesignator, WasArrayIndex); // At the point where the designators diverge, the comparison has a // specified value if: // - we are comparing array indices // - we are comparing fields of a union, or fields with the same access // Otherwise, the result is unspecified and thus the comparison is not a // constant expression. if (!WasArrayIndex && Mismatch < LHSDesignator.Entries.size() && Mismatch < RHSDesignator.Entries.size()) { const FieldDecl *LF = getAsField(LHSDesignator.Entries[Mismatch]); const FieldDecl *RF = getAsField(RHSDesignator.Entries[Mismatch]); if (!LF && !RF) Info.CCEDiag(E, diag::note_constexpr_pointer_comparison_base_classes); else if (!LF) Info.CCEDiag(E, diag::note_constexpr_pointer_comparison_base_field) << getAsBaseClass(LHSDesignator.Entries[Mismatch]) << RF->getParent() << RF; else if (!RF) Info.CCEDiag(E, diag::note_constexpr_pointer_comparison_base_field) << getAsBaseClass(RHSDesignator.Entries[Mismatch]) << LF->getParent() << LF; else if (!LF->getParent()->isUnion() && LF->getAccess() != RF->getAccess()) Info.CCEDiag(E, diag::note_constexpr_pointer_comparison_differing_access) << LF << LF->getAccess() << RF << RF->getAccess() << LF->getParent(); } } // The comparison here must be unsigned, and performed with the same // width as the pointer. unsigned PtrSize = Info.Ctx.getTypeSize(LHSTy); uint64_t CompareLHS = LHSOffset.getQuantity(); uint64_t CompareRHS = RHSOffset.getQuantity(); assert(PtrSize <= 64 && "Unexpected pointer width"); uint64_t Mask = ~0ULL >> (64 - PtrSize); CompareLHS &= Mask; CompareRHS &= Mask; // If there is a base and this is a relational operator, we can only // compare pointers within the object in question; otherwise, the result // depends on where the object is located in memory. if (!LHSValue.Base.isNull() && IsRelational) { QualType BaseTy = getType(LHSValue.Base); if (BaseTy->isIncompleteType()) return Error(E); CharUnits Size = Info.Ctx.getTypeSizeInChars(BaseTy); uint64_t OffsetLimit = Size.getQuantity(); if (CompareLHS > OffsetLimit || CompareRHS > OffsetLimit) return Error(E); } if (CompareLHS < CompareRHS) return Success(CCR::Less, E); if (CompareLHS > CompareRHS) return Success(CCR::Greater, E); return Success(CCR::Equal, E); } if (LHSTy->isMemberPointerType()) { assert(IsEquality && "unexpected member pointer operation"); assert(RHSTy->isMemberPointerType() && "invalid comparison"); MemberPtr LHSValue, RHSValue; bool LHSOK = EvaluateMemberPointer(E->getLHS(), LHSValue, Info); if (!LHSOK && !Info.noteFailure()) return false; if (!EvaluateMemberPointer(E->getRHS(), RHSValue, Info) || !LHSOK) return false; // C++11 [expr.eq]p2: // If both operands are null, they compare equal. Otherwise if only one is // null, they compare unequal. if (!LHSValue.getDecl() || !RHSValue.getDecl()) { bool Equal = !LHSValue.getDecl() && !RHSValue.getDecl(); return Success(Equal ? CCR::Equal : CCR::Nonequal, E); } // Otherwise if either is a pointer to a virtual member function, the // result is unspecified. if (const CXXMethodDecl *MD = dyn_cast(LHSValue.getDecl())) if (MD->isVirtual()) Info.CCEDiag(E, diag::note_constexpr_compare_virtual_mem_ptr) << MD; if (const CXXMethodDecl *MD = dyn_cast(RHSValue.getDecl())) if (MD->isVirtual()) Info.CCEDiag(E, diag::note_constexpr_compare_virtual_mem_ptr) << MD; // Otherwise they compare equal if and only if they would refer to the // same member of the same most derived object or the same subobject if // they were dereferenced with a hypothetical object of the associated // class type. bool Equal = LHSValue == RHSValue; return Success(Equal ? CCR::Equal : CCR::Nonequal, E); } if (LHSTy->isNullPtrType()) { assert(E->isComparisonOp() && "unexpected nullptr operation"); assert(RHSTy->isNullPtrType() && "missing pointer conversion"); // C++11 [expr.rel]p4, [expr.eq]p3: If two operands of type std::nullptr_t // are compared, the result is true of the operator is <=, >= or ==, and // false otherwise. return Success(CCR::Equal, E); } return DoAfter(); } bool RecordExprEvaluator::VisitBinCmp(const BinaryOperator *E) { if (!CheckLiteralType(Info, E)) return false; auto OnSuccess = [&](ComparisonCategoryResult ResKind, const BinaryOperator *E) { // Evaluation succeeded. Lookup the information for the comparison category // type and fetch the VarDecl for the result. const ComparisonCategoryInfo &CmpInfo = Info.Ctx.CompCategories.getInfoForType(E->getType()); const VarDecl *VD = CmpInfo.getValueInfo(CmpInfo.makeWeakResult(ResKind))->VD; // Check and evaluate the result as a constant expression. LValue LV; LV.set(VD); if (!handleLValueToRValueConversion(Info, E, E->getType(), LV, Result)) return false; return CheckConstantExpression(Info, E->getExprLoc(), E->getType(), Result); }; return EvaluateComparisonBinaryOperator(Info, E, OnSuccess, [&]() { return ExprEvaluatorBaseTy::VisitBinCmp(E); }); } bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) { // We don't call noteFailure immediately because the assignment happens after // we evaluate LHS and RHS. if (!Info.keepEvaluatingAfterFailure() && E->isAssignmentOp()) return Error(E); DelayedNoteFailureRAII MaybeNoteFailureLater(Info, E->isAssignmentOp()); if (DataRecursiveIntBinOpEvaluator::shouldEnqueue(E)) return DataRecursiveIntBinOpEvaluator(*this, Result).Traverse(E); assert((!E->getLHS()->getType()->isIntegralOrEnumerationType() || !E->getRHS()->getType()->isIntegralOrEnumerationType()) && "DataRecursiveIntBinOpEvaluator should have handled integral types"); if (E->isComparisonOp()) { // Evaluate builtin binary comparisons by evaluating them as C++2a three-way // comparisons and then translating the result. auto OnSuccess = [&](ComparisonCategoryResult ResKind, const BinaryOperator *E) { using CCR = ComparisonCategoryResult; bool IsEqual = ResKind == CCR::Equal, IsLess = ResKind == CCR::Less, IsGreater = ResKind == CCR::Greater; auto Op = E->getOpcode(); switch (Op) { default: llvm_unreachable("unsupported binary operator"); case BO_EQ: case BO_NE: return Success(IsEqual == (Op == BO_EQ), E); case BO_LT: return Success(IsLess, E); case BO_GT: return Success(IsGreater, E); case BO_LE: return Success(IsEqual || IsLess, E); case BO_GE: return Success(IsEqual || IsGreater, E); } }; return EvaluateComparisonBinaryOperator(Info, E, OnSuccess, [&]() { return ExprEvaluatorBaseTy::VisitBinaryOperator(E); }); } QualType LHSTy = E->getLHS()->getType(); QualType RHSTy = E->getRHS()->getType(); if (LHSTy->isPointerType() && RHSTy->isPointerType() && E->getOpcode() == BO_Sub) { LValue LHSValue, RHSValue; bool LHSOK = EvaluatePointer(E->getLHS(), LHSValue, Info); if (!LHSOK && !Info.noteFailure()) return false; if (!EvaluatePointer(E->getRHS(), RHSValue, Info) || !LHSOK) return false; // Reject differing bases from the normal codepath; we special-case // comparisons to null. if (!HasSameBase(LHSValue, RHSValue)) { // Handle &&A - &&B. if (!LHSValue.Offset.isZero() || !RHSValue.Offset.isZero()) return Error(E); const Expr *LHSExpr = LHSValue.Base.dyn_cast(); const Expr *RHSExpr = RHSValue.Base.dyn_cast(); if (!LHSExpr || !RHSExpr) return Error(E); const AddrLabelExpr *LHSAddrExpr = dyn_cast(LHSExpr); const AddrLabelExpr *RHSAddrExpr = dyn_cast(RHSExpr); if (!LHSAddrExpr || !RHSAddrExpr) return Error(E); // Make sure both labels come from the same function. if (LHSAddrExpr->getLabel()->getDeclContext() != RHSAddrExpr->getLabel()->getDeclContext()) return Error(E); return Success(APValue(LHSAddrExpr, RHSAddrExpr), E); } const CharUnits &LHSOffset = LHSValue.getLValueOffset(); const CharUnits &RHSOffset = RHSValue.getLValueOffset(); SubobjectDesignator &LHSDesignator = LHSValue.getLValueDesignator(); SubobjectDesignator &RHSDesignator = RHSValue.getLValueDesignator(); // C++11 [expr.add]p6: // Unless both pointers point to elements of the same array object, or // one past the last element of the array object, the behavior is // undefined. if (!LHSDesignator.Invalid && !RHSDesignator.Invalid && !AreElementsOfSameArray(getType(LHSValue.Base), LHSDesignator, RHSDesignator)) Info.CCEDiag(E, diag::note_constexpr_pointer_subtraction_not_same_array); QualType Type = E->getLHS()->getType(); QualType ElementType = Type->getAs()->getPointeeType(); CharUnits ElementSize; if (!HandleSizeof(Info, E->getExprLoc(), ElementType, ElementSize)) return false; // As an extension, a type may have zero size (empty struct or union in // C, array of zero length). Pointer subtraction in such cases has // undefined behavior, so is not constant. if (ElementSize.isZero()) { Info.FFDiag(E, diag::note_constexpr_pointer_subtraction_zero_size) << ElementType; return false; } // FIXME: LLVM and GCC both compute LHSOffset - RHSOffset at runtime, // and produce incorrect results when it overflows. Such behavior // appears to be non-conforming, but is common, so perhaps we should // assume the standard intended for such cases to be undefined behavior // and check for them. // Compute (LHSOffset - RHSOffset) / Size carefully, checking for // overflow in the final conversion to ptrdiff_t. APSInt LHS(llvm::APInt(65, (int64_t)LHSOffset.getQuantity(), true), false); APSInt RHS(llvm::APInt(65, (int64_t)RHSOffset.getQuantity(), true), false); APSInt ElemSize(llvm::APInt(65, (int64_t)ElementSize.getQuantity(), true), false); APSInt TrueResult = (LHS - RHS) / ElemSize; APSInt Result = TrueResult.trunc(Info.Ctx.getIntWidth(E->getType())); if (Result.extend(65) != TrueResult && !HandleOverflow(Info, E, TrueResult, E->getType())) return false; return Success(Result, E); } return ExprEvaluatorBaseTy::VisitBinaryOperator(E); } /// VisitUnaryExprOrTypeTraitExpr - Evaluate a sizeof, alignof or vec_step with /// a result as the expression's type. bool IntExprEvaluator::VisitUnaryExprOrTypeTraitExpr( const UnaryExprOrTypeTraitExpr *E) { switch(E->getKind()) { case UETT_PreferredAlignOf: case UETT_AlignOf: { if (E->isArgumentType()) return Success(GetAlignOfType(Info, E->getArgumentType(), E->getKind()), E); else return Success(GetAlignOfExpr(Info, E->getArgumentExpr(), E->getKind()), E); } case UETT_VecStep: { QualType Ty = E->getTypeOfArgument(); if (Ty->isVectorType()) { unsigned n = Ty->castAs()->getNumElements(); // The vec_step built-in functions that take a 3-component // vector return 4. (OpenCL 1.1 spec 6.11.12) if (n == 3) n = 4; return Success(n, E); } else return Success(1, E); } case UETT_SizeOf: { QualType SrcTy = E->getTypeOfArgument(); // C++ [expr.sizeof]p2: "When applied to a reference or a reference type, // the result is the size of the referenced type." if (const ReferenceType *Ref = SrcTy->getAs()) SrcTy = Ref->getPointeeType(); CharUnits Sizeof; if (!HandleSizeof(Info, E->getExprLoc(), SrcTy, Sizeof)) return false; return Success(Sizeof, E); } case UETT_OpenMPRequiredSimdAlign: assert(E->isArgumentType()); return Success( Info.Ctx.toCharUnitsFromBits( Info.Ctx.getOpenMPDefaultSimdAlign(E->getArgumentType())) .getQuantity(), E); } llvm_unreachable("unknown expr/type trait"); } bool IntExprEvaluator::VisitOffsetOfExpr(const OffsetOfExpr *OOE) { CharUnits Result; unsigned n = OOE->getNumComponents(); if (n == 0) return Error(OOE); QualType CurrentType = OOE->getTypeSourceInfo()->getType(); for (unsigned i = 0; i != n; ++i) { OffsetOfNode ON = OOE->getComponent(i); switch (ON.getKind()) { case OffsetOfNode::Array: { const Expr *Idx = OOE->getIndexExpr(ON.getArrayExprIndex()); APSInt IdxResult; if (!EvaluateInteger(Idx, IdxResult, Info)) return false; const ArrayType *AT = Info.Ctx.getAsArrayType(CurrentType); if (!AT) return Error(OOE); CurrentType = AT->getElementType(); CharUnits ElementSize = Info.Ctx.getTypeSizeInChars(CurrentType); Result += IdxResult.getSExtValue() * ElementSize; break; } case OffsetOfNode::Field: { FieldDecl *MemberDecl = ON.getField(); const RecordType *RT = CurrentType->getAs(); if (!RT) return Error(OOE); RecordDecl *RD = RT->getDecl(); if (RD->isInvalidDecl()) return false; const ASTRecordLayout &RL = Info.Ctx.getASTRecordLayout(RD); unsigned i = MemberDecl->getFieldIndex(); assert(i < RL.getFieldCount() && "offsetof field in wrong type"); Result += Info.Ctx.toCharUnitsFromBits(RL.getFieldOffset(i)); CurrentType = MemberDecl->getType().getNonReferenceType(); break; } case OffsetOfNode::Identifier: llvm_unreachable("dependent __builtin_offsetof"); case OffsetOfNode::Base: { CXXBaseSpecifier *BaseSpec = ON.getBase(); if (BaseSpec->isVirtual()) return Error(OOE); // Find the layout of the class whose base we are looking into. const RecordType *RT = CurrentType->getAs(); if (!RT) return Error(OOE); RecordDecl *RD = RT->getDecl(); if (RD->isInvalidDecl()) return false; const ASTRecordLayout &RL = Info.Ctx.getASTRecordLayout(RD); // Find the base class itself. CurrentType = BaseSpec->getType(); const RecordType *BaseRT = CurrentType->getAs(); if (!BaseRT) return Error(OOE); // Add the offset to the base. Result += RL.getBaseClassOffset(cast(BaseRT->getDecl())); break; } } } return Success(Result, OOE); } bool IntExprEvaluator::VisitUnaryOperator(const UnaryOperator *E) { switch (E->getOpcode()) { default: // Address, indirect, pre/post inc/dec, etc are not valid constant exprs. // See C99 6.6p3. return Error(E); case UO_Extension: // FIXME: Should extension allow i-c-e extension expressions in its scope? // If so, we could clear the diagnostic ID. return Visit(E->getSubExpr()); case UO_Plus: // The result is just the value. return Visit(E->getSubExpr()); case UO_Minus: { if (!Visit(E->getSubExpr())) return false; if (!Result.isInt()) return Error(E); const APSInt &Value = Result.getInt(); if (Value.isSigned() && Value.isMinSignedValue() && E->canOverflow() && !HandleOverflow(Info, E, -Value.extend(Value.getBitWidth() + 1), E->getType())) return false; return Success(-Value, E); } case UO_Not: { if (!Visit(E->getSubExpr())) return false; if (!Result.isInt()) return Error(E); return Success(~Result.getInt(), E); } case UO_LNot: { bool bres; if (!EvaluateAsBooleanCondition(E->getSubExpr(), bres, Info)) return false; return Success(!bres, E); } } } /// HandleCast - This is used to evaluate implicit or explicit casts where the /// result type is integer. bool IntExprEvaluator::VisitCastExpr(const CastExpr *E) { const Expr *SubExpr = E->getSubExpr(); QualType DestType = E->getType(); QualType SrcType = SubExpr->getType(); switch (E->getCastKind()) { case CK_BaseToDerived: case CK_DerivedToBase: case CK_UncheckedDerivedToBase: case CK_Dynamic: case CK_ToUnion: case CK_ArrayToPointerDecay: case CK_FunctionToPointerDecay: case CK_NullToPointer: case CK_NullToMemberPointer: case CK_BaseToDerivedMemberPointer: case CK_DerivedToBaseMemberPointer: case CK_ReinterpretMemberPointer: case CK_ConstructorConversion: case CK_IntegralToPointer: case CK_ToVoid: case CK_VectorSplat: case CK_IntegralToFloating: case CK_FloatingCast: case CK_CPointerToObjCPointerCast: case CK_BlockPointerToObjCPointerCast: case CK_AnyPointerToBlockPointerCast: case CK_ObjCObjectLValueCast: case CK_FloatingRealToComplex: case CK_FloatingComplexToReal: case CK_FloatingComplexCast: case CK_FloatingComplexToIntegralComplex: case CK_IntegralRealToComplex: case CK_IntegralComplexCast: case CK_IntegralComplexToFloatingComplex: case CK_BuiltinFnToFnPtr: case CK_ZeroToOCLOpaqueType: case CK_NonAtomicToAtomic: case CK_AddressSpaceConversion: case CK_IntToOCLSampler: case CK_FixedPointCast: case CK_IntegralToFixedPoint: llvm_unreachable("invalid cast kind for integral value"); case CK_BitCast: case CK_Dependent: case CK_LValueBitCast: case CK_ARCProduceObject: case CK_ARCConsumeObject: case CK_ARCReclaimReturnedObject: case CK_ARCExtendBlockObject: case CK_CopyAndAutoreleaseBlockObject: return Error(E); case CK_UserDefinedConversion: case CK_LValueToRValue: case CK_AtomicToNonAtomic: case CK_NoOp: case CK_LValueToRValueBitCast: return ExprEvaluatorBaseTy::VisitCastExpr(E); case CK_MemberPointerToBoolean: case CK_PointerToBoolean: case CK_IntegralToBoolean: case CK_FloatingToBoolean: case CK_BooleanToSignedIntegral: case CK_FloatingComplexToBoolean: case CK_IntegralComplexToBoolean: { bool BoolResult; if (!EvaluateAsBooleanCondition(SubExpr, BoolResult, Info)) return false; uint64_t IntResult = BoolResult; if (BoolResult && E->getCastKind() == CK_BooleanToSignedIntegral) IntResult = (uint64_t)-1; return Success(IntResult, E); } case CK_FixedPointToIntegral: { APFixedPoint Src(Info.Ctx.getFixedPointSemantics(SrcType)); if (!EvaluateFixedPoint(SubExpr, Src, Info)) return false; bool Overflowed; llvm::APSInt Result = Src.convertToInt( Info.Ctx.getIntWidth(DestType), DestType->isSignedIntegerOrEnumerationType(), &Overflowed); if (Overflowed && !HandleOverflow(Info, E, Result, DestType)) return false; return Success(Result, E); } case CK_FixedPointToBoolean: { // Unsigned padding does not affect this. APValue Val; if (!Evaluate(Val, Info, SubExpr)) return false; return Success(Val.getFixedPoint().getBoolValue(), E); } case CK_IntegralCast: { if (!Visit(SubExpr)) return false; if (!Result.isInt()) { // Allow casts of address-of-label differences if they are no-ops // or narrowing. (The narrowing case isn't actually guaranteed to // be constant-evaluatable except in some narrow cases which are hard // to detect here. We let it through on the assumption the user knows // what they are doing.) if (Result.isAddrLabelDiff()) return Info.Ctx.getTypeSize(DestType) <= Info.Ctx.getTypeSize(SrcType); // Only allow casts of lvalues if they are lossless. return Info.Ctx.getTypeSize(DestType) == Info.Ctx.getTypeSize(SrcType); } return Success(HandleIntToIntCast(Info, E, DestType, SrcType, Result.getInt()), E); } case CK_PointerToIntegral: { CCEDiag(E, diag::note_constexpr_invalid_cast) << 2; LValue LV; if (!EvaluatePointer(SubExpr, LV, Info)) return false; if (LV.getLValueBase()) { // Only allow based lvalue casts if they are lossless. // FIXME: Allow a larger integer size than the pointer size, and allow // narrowing back down to pointer width in subsequent integral casts. // FIXME: Check integer type's active bits, not its type size. if (Info.Ctx.getTypeSize(DestType) != Info.Ctx.getTypeSize(SrcType)) return Error(E); LV.Designator.setInvalid(); LV.moveInto(Result); return true; } APSInt AsInt; APValue V; LV.moveInto(V); if (!V.toIntegralConstant(AsInt, SrcType, Info.Ctx)) llvm_unreachable("Can't cast this!"); return Success(HandleIntToIntCast(Info, E, DestType, SrcType, AsInt), E); } case CK_IntegralComplexToReal: { ComplexValue C; if (!EvaluateComplex(SubExpr, C, Info)) return false; return Success(C.getComplexIntReal(), E); } case CK_FloatingToIntegral: { APFloat F(0.0); if (!EvaluateFloat(SubExpr, F, Info)) return false; APSInt Value; if (!HandleFloatToIntCast(Info, E, SrcType, F, DestType, Value)) return false; return Success(Value, E); } } llvm_unreachable("unknown cast resulting in integral value"); } bool IntExprEvaluator::VisitUnaryReal(const UnaryOperator *E) { if (E->getSubExpr()->getType()->isAnyComplexType()) { ComplexValue LV; if (!EvaluateComplex(E->getSubExpr(), LV, Info)) return false; if (!LV.isComplexInt()) return Error(E); return Success(LV.getComplexIntReal(), E); } return Visit(E->getSubExpr()); } bool IntExprEvaluator::VisitUnaryImag(const UnaryOperator *E) { if (E->getSubExpr()->getType()->isComplexIntegerType()) { ComplexValue LV; if (!EvaluateComplex(E->getSubExpr(), LV, Info)) return false; if (!LV.isComplexInt()) return Error(E); return Success(LV.getComplexIntImag(), E); } VisitIgnoredValue(E->getSubExpr()); return Success(0, E); } bool IntExprEvaluator::VisitSizeOfPackExpr(const SizeOfPackExpr *E) { return Success(E->getPackLength(), E); } bool IntExprEvaluator::VisitCXXNoexceptExpr(const CXXNoexceptExpr *E) { return Success(E->getValue(), E); } bool FixedPointExprEvaluator::VisitUnaryOperator(const UnaryOperator *E) { switch (E->getOpcode()) { default: // Invalid unary operators return Error(E); case UO_Plus: // The result is just the value. return Visit(E->getSubExpr()); case UO_Minus: { if (!Visit(E->getSubExpr())) return false; if (!Result.isFixedPoint()) return Error(E); bool Overflowed; APFixedPoint Negated = Result.getFixedPoint().negate(&Overflowed); if (Overflowed && !HandleOverflow(Info, E, Negated, E->getType())) return false; return Success(Negated, E); } case UO_LNot: { bool bres; if (!EvaluateAsBooleanCondition(E->getSubExpr(), bres, Info)) return false; return Success(!bres, E); } } } bool FixedPointExprEvaluator::VisitCastExpr(const CastExpr *E) { const Expr *SubExpr = E->getSubExpr(); QualType DestType = E->getType(); assert(DestType->isFixedPointType() && "Expected destination type to be a fixed point type"); auto DestFXSema = Info.Ctx.getFixedPointSemantics(DestType); switch (E->getCastKind()) { case CK_FixedPointCast: { APFixedPoint Src(Info.Ctx.getFixedPointSemantics(SubExpr->getType())); if (!EvaluateFixedPoint(SubExpr, Src, Info)) return false; bool Overflowed; APFixedPoint Result = Src.convert(DestFXSema, &Overflowed); if (Overflowed && !HandleOverflow(Info, E, Result, DestType)) return false; return Success(Result, E); } case CK_IntegralToFixedPoint: { APSInt Src; if (!EvaluateInteger(SubExpr, Src, Info)) return false; bool Overflowed; APFixedPoint IntResult = APFixedPoint::getFromIntValue( Src, Info.Ctx.getFixedPointSemantics(DestType), &Overflowed); if (Overflowed && !HandleOverflow(Info, E, IntResult, DestType)) return false; return Success(IntResult, E); } case CK_NoOp: case CK_LValueToRValue: return ExprEvaluatorBaseTy::VisitCastExpr(E); default: return Error(E); } } bool FixedPointExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) { const Expr *LHS = E->getLHS(); const Expr *RHS = E->getRHS(); FixedPointSemantics ResultFXSema = Info.Ctx.getFixedPointSemantics(E->getType()); APFixedPoint LHSFX(Info.Ctx.getFixedPointSemantics(LHS->getType())); if (!EvaluateFixedPointOrInteger(LHS, LHSFX, Info)) return false; APFixedPoint RHSFX(Info.Ctx.getFixedPointSemantics(RHS->getType())); if (!EvaluateFixedPointOrInteger(RHS, RHSFX, Info)) return false; switch (E->getOpcode()) { case BO_Add: { bool AddOverflow, ConversionOverflow; APFixedPoint Result = LHSFX.add(RHSFX, &AddOverflow) .convert(ResultFXSema, &ConversionOverflow); if ((AddOverflow || ConversionOverflow) && !HandleOverflow(Info, E, Result, E->getType())) return false; return Success(Result, E); } default: return false; } llvm_unreachable("Should've exited before this"); } //===----------------------------------------------------------------------===// // Float Evaluation //===----------------------------------------------------------------------===// namespace { class FloatExprEvaluator : public ExprEvaluatorBase { APFloat &Result; public: FloatExprEvaluator(EvalInfo &info, APFloat &result) : ExprEvaluatorBaseTy(info), Result(result) {} bool Success(const APValue &V, const Expr *e) { Result = V.getFloat(); return true; } bool ZeroInitialization(const Expr *E) { Result = APFloat::getZero(Info.Ctx.getFloatTypeSemantics(E->getType())); return true; } bool VisitCallExpr(const CallExpr *E); bool VisitUnaryOperator(const UnaryOperator *E); bool VisitBinaryOperator(const BinaryOperator *E); bool VisitFloatingLiteral(const FloatingLiteral *E); bool VisitCastExpr(const CastExpr *E); bool VisitUnaryReal(const UnaryOperator *E); bool VisitUnaryImag(const UnaryOperator *E); // FIXME: Missing: array subscript of vector, member of vector }; } // end anonymous namespace static bool EvaluateFloat(const Expr* E, APFloat& Result, EvalInfo &Info) { assert(E->isRValue() && E->getType()->isRealFloatingType()); return FloatExprEvaluator(Info, Result).Visit(E); } static bool TryEvaluateBuiltinNaN(const ASTContext &Context, QualType ResultTy, const Expr *Arg, bool SNaN, llvm::APFloat &Result) { const StringLiteral *S = dyn_cast(Arg->IgnoreParenCasts()); if (!S) return false; const llvm::fltSemantics &Sem = Context.getFloatTypeSemantics(ResultTy); llvm::APInt fill; // Treat empty strings as if they were zero. if (S->getString().empty()) fill = llvm::APInt(32, 0); else if (S->getString().getAsInteger(0, fill)) return false; if (Context.getTargetInfo().isNan2008()) { if (SNaN) Result = llvm::APFloat::getSNaN(Sem, false, &fill); else Result = llvm::APFloat::getQNaN(Sem, false, &fill); } else { // Prior to IEEE 754-2008, architectures were allowed to choose whether // the first bit of their significand was set for qNaN or sNaN. MIPS chose // a different encoding to what became a standard in 2008, and for pre- // 2008 revisions, MIPS interpreted sNaN-2008 as qNan and qNaN-2008 as // sNaN. This is now known as "legacy NaN" encoding. if (SNaN) Result = llvm::APFloat::getQNaN(Sem, false, &fill); else Result = llvm::APFloat::getSNaN(Sem, false, &fill); } return true; } bool FloatExprEvaluator::VisitCallExpr(const CallExpr *E) { switch (E->getBuiltinCallee()) { default: return ExprEvaluatorBaseTy::VisitCallExpr(E); case Builtin::BI__builtin_huge_val: case Builtin::BI__builtin_huge_valf: case Builtin::BI__builtin_huge_vall: case Builtin::BI__builtin_huge_valf128: case Builtin::BI__builtin_inf: case Builtin::BI__builtin_inff: case Builtin::BI__builtin_infl: case Builtin::BI__builtin_inff128: { const llvm::fltSemantics &Sem = Info.Ctx.getFloatTypeSemantics(E->getType()); Result = llvm::APFloat::getInf(Sem); return true; } case Builtin::BI__builtin_nans: case Builtin::BI__builtin_nansf: case Builtin::BI__builtin_nansl: case Builtin::BI__builtin_nansf128: if (!TryEvaluateBuiltinNaN(Info.Ctx, E->getType(), E->getArg(0), true, Result)) return Error(E); return true; case Builtin::BI__builtin_nan: case Builtin::BI__builtin_nanf: case Builtin::BI__builtin_nanl: case Builtin::BI__builtin_nanf128: // If this is __builtin_nan() turn this into a nan, otherwise we // can't constant fold it. if (!TryEvaluateBuiltinNaN(Info.Ctx, E->getType(), E->getArg(0), false, Result)) return Error(E); return true; case Builtin::BI__builtin_fabs: case Builtin::BI__builtin_fabsf: case Builtin::BI__builtin_fabsl: case Builtin::BI__builtin_fabsf128: if (!EvaluateFloat(E->getArg(0), Result, Info)) return false; if (Result.isNegative()) Result.changeSign(); return true; // FIXME: Builtin::BI__builtin_powi // FIXME: Builtin::BI__builtin_powif // FIXME: Builtin::BI__builtin_powil case Builtin::BI__builtin_copysign: case Builtin::BI__builtin_copysignf: case Builtin::BI__builtin_copysignl: case Builtin::BI__builtin_copysignf128: { APFloat RHS(0.); if (!EvaluateFloat(E->getArg(0), Result, Info) || !EvaluateFloat(E->getArg(1), RHS, Info)) return false; Result.copySign(RHS); return true; } } } bool FloatExprEvaluator::VisitUnaryReal(const UnaryOperator *E) { if (E->getSubExpr()->getType()->isAnyComplexType()) { ComplexValue CV; if (!EvaluateComplex(E->getSubExpr(), CV, Info)) return false; Result = CV.FloatReal; return true; } return Visit(E->getSubExpr()); } bool FloatExprEvaluator::VisitUnaryImag(const UnaryOperator *E) { if (E->getSubExpr()->getType()->isAnyComplexType()) { ComplexValue CV; if (!EvaluateComplex(E->getSubExpr(), CV, Info)) return false; Result = CV.FloatImag; return true; } VisitIgnoredValue(E->getSubExpr()); const llvm::fltSemantics &Sem = Info.Ctx.getFloatTypeSemantics(E->getType()); Result = llvm::APFloat::getZero(Sem); return true; } bool FloatExprEvaluator::VisitUnaryOperator(const UnaryOperator *E) { switch (E->getOpcode()) { default: return Error(E); case UO_Plus: return EvaluateFloat(E->getSubExpr(), Result, Info); case UO_Minus: if (!EvaluateFloat(E->getSubExpr(), Result, Info)) return false; Result.changeSign(); return true; } } bool FloatExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) { if (E->isPtrMemOp() || E->isAssignmentOp() || E->getOpcode() == BO_Comma) return ExprEvaluatorBaseTy::VisitBinaryOperator(E); APFloat RHS(0.0); bool LHSOK = EvaluateFloat(E->getLHS(), Result, Info); if (!LHSOK && !Info.noteFailure()) return false; return EvaluateFloat(E->getRHS(), RHS, Info) && LHSOK && handleFloatFloatBinOp(Info, E, Result, E->getOpcode(), RHS); } bool FloatExprEvaluator::VisitFloatingLiteral(const FloatingLiteral *E) { Result = E->getValue(); return true; } bool FloatExprEvaluator::VisitCastExpr(const CastExpr *E) { const Expr* SubExpr = E->getSubExpr(); switch (E->getCastKind()) { default: return ExprEvaluatorBaseTy::VisitCastExpr(E); case CK_IntegralToFloating: { APSInt IntResult; return EvaluateInteger(SubExpr, IntResult, Info) && HandleIntToFloatCast(Info, E, SubExpr->getType(), IntResult, E->getType(), Result); } case CK_FloatingCast: { if (!Visit(SubExpr)) return false; return HandleFloatToFloatCast(Info, E, SubExpr->getType(), E->getType(), Result); } case CK_FloatingComplexToReal: { ComplexValue V; if (!EvaluateComplex(SubExpr, V, Info)) return false; Result = V.getComplexFloatReal(); return true; } } } //===----------------------------------------------------------------------===// // Complex Evaluation (for float and integer) //===----------------------------------------------------------------------===// namespace { class ComplexExprEvaluator : public ExprEvaluatorBase { ComplexValue &Result; public: ComplexExprEvaluator(EvalInfo &info, ComplexValue &Result) : ExprEvaluatorBaseTy(info), Result(Result) {} bool Success(const APValue &V, const Expr *e) { Result.setFrom(V); return true; } bool ZeroInitialization(const Expr *E); //===--------------------------------------------------------------------===// // Visitor Methods //===--------------------------------------------------------------------===// bool VisitImaginaryLiteral(const ImaginaryLiteral *E); bool VisitCastExpr(const CastExpr *E); bool VisitBinaryOperator(const BinaryOperator *E); bool VisitUnaryOperator(const UnaryOperator *E); bool VisitInitListExpr(const InitListExpr *E); }; } // end anonymous namespace static bool EvaluateComplex(const Expr *E, ComplexValue &Result, EvalInfo &Info) { assert(E->isRValue() && E->getType()->isAnyComplexType()); return ComplexExprEvaluator(Info, Result).Visit(E); } bool ComplexExprEvaluator::ZeroInitialization(const Expr *E) { QualType ElemTy = E->getType()->castAs()->getElementType(); if (ElemTy->isRealFloatingType()) { Result.makeComplexFloat(); APFloat Zero = APFloat::getZero(Info.Ctx.getFloatTypeSemantics(ElemTy)); Result.FloatReal = Zero; Result.FloatImag = Zero; } else { Result.makeComplexInt(); APSInt Zero = Info.Ctx.MakeIntValue(0, ElemTy); Result.IntReal = Zero; Result.IntImag = Zero; } return true; } bool ComplexExprEvaluator::VisitImaginaryLiteral(const ImaginaryLiteral *E) { const Expr* SubExpr = E->getSubExpr(); if (SubExpr->getType()->isRealFloatingType()) { Result.makeComplexFloat(); APFloat &Imag = Result.FloatImag; if (!EvaluateFloat(SubExpr, Imag, Info)) return false; Result.FloatReal = APFloat(Imag.getSemantics()); return true; } else { assert(SubExpr->getType()->isIntegerType() && "Unexpected imaginary literal."); Result.makeComplexInt(); APSInt &Imag = Result.IntImag; if (!EvaluateInteger(SubExpr, Imag, Info)) return false; Result.IntReal = APSInt(Imag.getBitWidth(), !Imag.isSigned()); return true; } } bool ComplexExprEvaluator::VisitCastExpr(const CastExpr *E) { switch (E->getCastKind()) { case CK_BitCast: case CK_BaseToDerived: case CK_DerivedToBase: case CK_UncheckedDerivedToBase: case CK_Dynamic: case CK_ToUnion: case CK_ArrayToPointerDecay: case CK_FunctionToPointerDecay: case CK_NullToPointer: case CK_NullToMemberPointer: case CK_BaseToDerivedMemberPointer: case CK_DerivedToBaseMemberPointer: case CK_MemberPointerToBoolean: case CK_ReinterpretMemberPointer: case CK_ConstructorConversion: case CK_IntegralToPointer: case CK_PointerToIntegral: case CK_PointerToBoolean: case CK_ToVoid: case CK_VectorSplat: case CK_IntegralCast: case CK_BooleanToSignedIntegral: case CK_IntegralToBoolean: case CK_IntegralToFloating: case CK_FloatingToIntegral: case CK_FloatingToBoolean: case CK_FloatingCast: case CK_CPointerToObjCPointerCast: case CK_BlockPointerToObjCPointerCast: case CK_AnyPointerToBlockPointerCast: case CK_ObjCObjectLValueCast: case CK_FloatingComplexToReal: case CK_FloatingComplexToBoolean: case CK_IntegralComplexToReal: case CK_IntegralComplexToBoolean: case CK_ARCProduceObject: case CK_ARCConsumeObject: case CK_ARCReclaimReturnedObject: case CK_ARCExtendBlockObject: case CK_CopyAndAutoreleaseBlockObject: case CK_BuiltinFnToFnPtr: case CK_ZeroToOCLOpaqueType: case CK_NonAtomicToAtomic: case CK_AddressSpaceConversion: case CK_IntToOCLSampler: case CK_FixedPointCast: case CK_FixedPointToBoolean: case CK_FixedPointToIntegral: case CK_IntegralToFixedPoint: llvm_unreachable("invalid cast kind for complex value"); case CK_LValueToRValue: case CK_AtomicToNonAtomic: case CK_NoOp: case CK_LValueToRValueBitCast: return ExprEvaluatorBaseTy::VisitCastExpr(E); case CK_Dependent: case CK_LValueBitCast: case CK_UserDefinedConversion: return Error(E); case CK_FloatingRealToComplex: { APFloat &Real = Result.FloatReal; if (!EvaluateFloat(E->getSubExpr(), Real, Info)) return false; Result.makeComplexFloat(); Result.FloatImag = APFloat(Real.getSemantics()); return true; } case CK_FloatingComplexCast: { if (!Visit(E->getSubExpr())) return false; QualType To = E->getType()->getAs()->getElementType(); QualType From = E->getSubExpr()->getType()->getAs()->getElementType(); return HandleFloatToFloatCast(Info, E, From, To, Result.FloatReal) && HandleFloatToFloatCast(Info, E, From, To, Result.FloatImag); } case CK_FloatingComplexToIntegralComplex: { if (!Visit(E->getSubExpr())) return false; QualType To = E->getType()->getAs()->getElementType(); QualType From = E->getSubExpr()->getType()->getAs()->getElementType(); Result.makeComplexInt(); return HandleFloatToIntCast(Info, E, From, Result.FloatReal, To, Result.IntReal) && HandleFloatToIntCast(Info, E, From, Result.FloatImag, To, Result.IntImag); } case CK_IntegralRealToComplex: { APSInt &Real = Result.IntReal; if (!EvaluateInteger(E->getSubExpr(), Real, Info)) return false; Result.makeComplexInt(); Result.IntImag = APSInt(Real.getBitWidth(), !Real.isSigned()); return true; } case CK_IntegralComplexCast: { if (!Visit(E->getSubExpr())) return false; QualType To = E->getType()->getAs()->getElementType(); QualType From = E->getSubExpr()->getType()->getAs()->getElementType(); Result.IntReal = HandleIntToIntCast(Info, E, To, From, Result.IntReal); Result.IntImag = HandleIntToIntCast(Info, E, To, From, Result.IntImag); return true; } case CK_IntegralComplexToFloatingComplex: { if (!Visit(E->getSubExpr())) return false; QualType To = E->getType()->castAs()->getElementType(); QualType From = E->getSubExpr()->getType()->castAs()->getElementType(); Result.makeComplexFloat(); return HandleIntToFloatCast(Info, E, From, Result.IntReal, To, Result.FloatReal) && HandleIntToFloatCast(Info, E, From, Result.IntImag, To, Result.FloatImag); } } llvm_unreachable("unknown cast resulting in complex value"); } bool ComplexExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) { if (E->isPtrMemOp() || E->isAssignmentOp() || E->getOpcode() == BO_Comma) return ExprEvaluatorBaseTy::VisitBinaryOperator(E); // Track whether the LHS or RHS is real at the type system level. When this is // the case we can simplify our evaluation strategy. bool LHSReal = false, RHSReal = false; bool LHSOK; if (E->getLHS()->getType()->isRealFloatingType()) { LHSReal = true; APFloat &Real = Result.FloatReal; LHSOK = EvaluateFloat(E->getLHS(), Real, Info); if (LHSOK) { Result.makeComplexFloat(); Result.FloatImag = APFloat(Real.getSemantics()); } } else { LHSOK = Visit(E->getLHS()); } if (!LHSOK && !Info.noteFailure()) return false; ComplexValue RHS; if (E->getRHS()->getType()->isRealFloatingType()) { RHSReal = true; APFloat &Real = RHS.FloatReal; if (!EvaluateFloat(E->getRHS(), Real, Info) || !LHSOK) return false; RHS.makeComplexFloat(); RHS.FloatImag = APFloat(Real.getSemantics()); } else if (!EvaluateComplex(E->getRHS(), RHS, Info) || !LHSOK) return false; assert(!(LHSReal && RHSReal) && "Cannot have both operands of a complex operation be real."); switch (E->getOpcode()) { default: return Error(E); case BO_Add: if (Result.isComplexFloat()) { Result.getComplexFloatReal().add(RHS.getComplexFloatReal(), APFloat::rmNearestTiesToEven); if (LHSReal) Result.getComplexFloatImag() = RHS.getComplexFloatImag(); else if (!RHSReal) Result.getComplexFloatImag().add(RHS.getComplexFloatImag(), APFloat::rmNearestTiesToEven); } else { Result.getComplexIntReal() += RHS.getComplexIntReal(); Result.getComplexIntImag() += RHS.getComplexIntImag(); } break; case BO_Sub: if (Result.isComplexFloat()) { Result.getComplexFloatReal().subtract(RHS.getComplexFloatReal(), APFloat::rmNearestTiesToEven); if (LHSReal) { Result.getComplexFloatImag() = RHS.getComplexFloatImag(); Result.getComplexFloatImag().changeSign(); } else if (!RHSReal) { Result.getComplexFloatImag().subtract(RHS.getComplexFloatImag(), APFloat::rmNearestTiesToEven); } } else { Result.getComplexIntReal() -= RHS.getComplexIntReal(); Result.getComplexIntImag() -= RHS.getComplexIntImag(); } break; case BO_Mul: if (Result.isComplexFloat()) { // This is an implementation of complex multiplication according to the // constraints laid out in C11 Annex G. The implementation uses the // following naming scheme: // (a + ib) * (c + id) ComplexValue LHS = Result; APFloat &A = LHS.getComplexFloatReal(); APFloat &B = LHS.getComplexFloatImag(); APFloat &C = RHS.getComplexFloatReal(); APFloat &D = RHS.getComplexFloatImag(); APFloat &ResR = Result.getComplexFloatReal(); APFloat &ResI = Result.getComplexFloatImag(); if (LHSReal) { assert(!RHSReal && "Cannot have two real operands for a complex op!"); ResR = A * C; ResI = A * D; } else if (RHSReal) { ResR = C * A; ResI = C * B; } else { // In the fully general case, we need to handle NaNs and infinities // robustly. APFloat AC = A * C; APFloat BD = B * D; APFloat AD = A * D; APFloat BC = B * C; ResR = AC - BD; ResI = AD + BC; if (ResR.isNaN() && ResI.isNaN()) { bool Recalc = false; if (A.isInfinity() || B.isInfinity()) { A = APFloat::copySign( APFloat(A.getSemantics(), A.isInfinity() ? 1 : 0), A); B = APFloat::copySign( APFloat(B.getSemantics(), B.isInfinity() ? 1 : 0), B); if (C.isNaN()) C = APFloat::copySign(APFloat(C.getSemantics()), C); if (D.isNaN()) D = APFloat::copySign(APFloat(D.getSemantics()), D); Recalc = true; } if (C.isInfinity() || D.isInfinity()) { C = APFloat::copySign( APFloat(C.getSemantics(), C.isInfinity() ? 1 : 0), C); D = APFloat::copySign( APFloat(D.getSemantics(), D.isInfinity() ? 1 : 0), D); if (A.isNaN()) A = APFloat::copySign(APFloat(A.getSemantics()), A); if (B.isNaN()) B = APFloat::copySign(APFloat(B.getSemantics()), B); Recalc = true; } if (!Recalc && (AC.isInfinity() || BD.isInfinity() || AD.isInfinity() || BC.isInfinity())) { if (A.isNaN()) A = APFloat::copySign(APFloat(A.getSemantics()), A); if (B.isNaN()) B = APFloat::copySign(APFloat(B.getSemantics()), B); if (C.isNaN()) C = APFloat::copySign(APFloat(C.getSemantics()), C); if (D.isNaN()) D = APFloat::copySign(APFloat(D.getSemantics()), D); Recalc = true; } if (Recalc) { ResR = APFloat::getInf(A.getSemantics()) * (A * C - B * D); ResI = APFloat::getInf(A.getSemantics()) * (A * D + B * C); } } } } else { ComplexValue LHS = Result; Result.getComplexIntReal() = (LHS.getComplexIntReal() * RHS.getComplexIntReal() - LHS.getComplexIntImag() * RHS.getComplexIntImag()); Result.getComplexIntImag() = (LHS.getComplexIntReal() * RHS.getComplexIntImag() + LHS.getComplexIntImag() * RHS.getComplexIntReal()); } break; case BO_Div: if (Result.isComplexFloat()) { // This is an implementation of complex division according to the // constraints laid out in C11 Annex G. The implementation uses the // following naming scheme: // (a + ib) / (c + id) ComplexValue LHS = Result; APFloat &A = LHS.getComplexFloatReal(); APFloat &B = LHS.getComplexFloatImag(); APFloat &C = RHS.getComplexFloatReal(); APFloat &D = RHS.getComplexFloatImag(); APFloat &ResR = Result.getComplexFloatReal(); APFloat &ResI = Result.getComplexFloatImag(); if (RHSReal) { ResR = A / C; ResI = B / C; } else { if (LHSReal) { // No real optimizations we can do here, stub out with zero. B = APFloat::getZero(A.getSemantics()); } int DenomLogB = 0; APFloat MaxCD = maxnum(abs(C), abs(D)); if (MaxCD.isFinite()) { DenomLogB = ilogb(MaxCD); C = scalbn(C, -DenomLogB, APFloat::rmNearestTiesToEven); D = scalbn(D, -DenomLogB, APFloat::rmNearestTiesToEven); } APFloat Denom = C * C + D * D; ResR = scalbn((A * C + B * D) / Denom, -DenomLogB, APFloat::rmNearestTiesToEven); ResI = scalbn((B * C - A * D) / Denom, -DenomLogB, APFloat::rmNearestTiesToEven); if (ResR.isNaN() && ResI.isNaN()) { if (Denom.isPosZero() && (!A.isNaN() || !B.isNaN())) { ResR = APFloat::getInf(ResR.getSemantics(), C.isNegative()) * A; ResI = APFloat::getInf(ResR.getSemantics(), C.isNegative()) * B; } else if ((A.isInfinity() || B.isInfinity()) && C.isFinite() && D.isFinite()) { A = APFloat::copySign( APFloat(A.getSemantics(), A.isInfinity() ? 1 : 0), A); B = APFloat::copySign( APFloat(B.getSemantics(), B.isInfinity() ? 1 : 0), B); ResR = APFloat::getInf(ResR.getSemantics()) * (A * C + B * D); ResI = APFloat::getInf(ResI.getSemantics()) * (B * C - A * D); } else if (MaxCD.isInfinity() && A.isFinite() && B.isFinite()) { C = APFloat::copySign( APFloat(C.getSemantics(), C.isInfinity() ? 1 : 0), C); D = APFloat::copySign( APFloat(D.getSemantics(), D.isInfinity() ? 1 : 0), D); ResR = APFloat::getZero(ResR.getSemantics()) * (A * C + B * D); ResI = APFloat::getZero(ResI.getSemantics()) * (B * C - A * D); } } } } else { if (RHS.getComplexIntReal() == 0 && RHS.getComplexIntImag() == 0) return Error(E, diag::note_expr_divide_by_zero); ComplexValue LHS = Result; APSInt Den = RHS.getComplexIntReal() * RHS.getComplexIntReal() + RHS.getComplexIntImag() * RHS.getComplexIntImag(); Result.getComplexIntReal() = (LHS.getComplexIntReal() * RHS.getComplexIntReal() + LHS.getComplexIntImag() * RHS.getComplexIntImag()) / Den; Result.getComplexIntImag() = (LHS.getComplexIntImag() * RHS.getComplexIntReal() - LHS.getComplexIntReal() * RHS.getComplexIntImag()) / Den; } break; } return true; } bool ComplexExprEvaluator::VisitUnaryOperator(const UnaryOperator *E) { // Get the operand value into 'Result'. if (!Visit(E->getSubExpr())) return false; switch (E->getOpcode()) { default: return Error(E); case UO_Extension: return true; case UO_Plus: // The result is always just the subexpr. return true; case UO_Minus: if (Result.isComplexFloat()) { Result.getComplexFloatReal().changeSign(); Result.getComplexFloatImag().changeSign(); } else { Result.getComplexIntReal() = -Result.getComplexIntReal(); Result.getComplexIntImag() = -Result.getComplexIntImag(); } return true; case UO_Not: if (Result.isComplexFloat()) Result.getComplexFloatImag().changeSign(); else Result.getComplexIntImag() = -Result.getComplexIntImag(); return true; } } bool ComplexExprEvaluator::VisitInitListExpr(const InitListExpr *E) { if (E->getNumInits() == 2) { if (E->getType()->isComplexType()) { Result.makeComplexFloat(); if (!EvaluateFloat(E->getInit(0), Result.FloatReal, Info)) return false; if (!EvaluateFloat(E->getInit(1), Result.FloatImag, Info)) return false; } else { Result.makeComplexInt(); if (!EvaluateInteger(E->getInit(0), Result.IntReal, Info)) return false; if (!EvaluateInteger(E->getInit(1), Result.IntImag, Info)) return false; } return true; } return ExprEvaluatorBaseTy::VisitInitListExpr(E); } //===----------------------------------------------------------------------===// // Atomic expression evaluation, essentially just handling the NonAtomicToAtomic // implicit conversion. //===----------------------------------------------------------------------===// namespace { class AtomicExprEvaluator : public ExprEvaluatorBase { const LValue *This; APValue &Result; public: AtomicExprEvaluator(EvalInfo &Info, const LValue *This, APValue &Result) : ExprEvaluatorBaseTy(Info), This(This), Result(Result) {} bool Success(const APValue &V, const Expr *E) { Result = V; return true; } bool ZeroInitialization(const Expr *E) { ImplicitValueInitExpr VIE( E->getType()->castAs()->getValueType()); // For atomic-qualified class (and array) types in C++, initialize the // _Atomic-wrapped subobject directly, in-place. return This ? EvaluateInPlace(Result, Info, *This, &VIE) : Evaluate(Result, Info, &VIE); } bool VisitCastExpr(const CastExpr *E) { switch (E->getCastKind()) { default: return ExprEvaluatorBaseTy::VisitCastExpr(E); case CK_NonAtomicToAtomic: return This ? EvaluateInPlace(Result, Info, *This, E->getSubExpr()) : Evaluate(Result, Info, E->getSubExpr()); } } }; } // end anonymous namespace static bool EvaluateAtomic(const Expr *E, const LValue *This, APValue &Result, EvalInfo &Info) { assert(E->isRValue() && E->getType()->isAtomicType()); return AtomicExprEvaluator(Info, This, Result).Visit(E); } //===----------------------------------------------------------------------===// // Void expression evaluation, primarily for a cast to void on the LHS of a // comma operator //===----------------------------------------------------------------------===// namespace { class VoidExprEvaluator : public ExprEvaluatorBase { public: VoidExprEvaluator(EvalInfo &Info) : ExprEvaluatorBaseTy(Info) {} bool Success(const APValue &V, const Expr *e) { return true; } bool ZeroInitialization(const Expr *E) { return true; } bool VisitCastExpr(const CastExpr *E) { switch (E->getCastKind()) { default: return ExprEvaluatorBaseTy::VisitCastExpr(E); case CK_ToVoid: VisitIgnoredValue(E->getSubExpr()); return true; } } bool VisitCallExpr(const CallExpr *E) { switch (E->getBuiltinCallee()) { default: return ExprEvaluatorBaseTy::VisitCallExpr(E); case Builtin::BI__assume: case Builtin::BI__builtin_assume: // The argument is not evaluated! return true; } } }; } // end anonymous namespace static bool EvaluateVoid(const Expr *E, EvalInfo &Info) { assert(E->isRValue() && E->getType()->isVoidType()); return VoidExprEvaluator(Info).Visit(E); } //===----------------------------------------------------------------------===// // Top level Expr::EvaluateAsRValue method. //===----------------------------------------------------------------------===// static bool Evaluate(APValue &Result, EvalInfo &Info, const Expr *E) { // In C, function designators are not lvalues, but we evaluate them as if they // are. QualType T = E->getType(); if (E->isGLValue() || T->isFunctionType()) { LValue LV; if (!EvaluateLValue(E, LV, Info)) return false; LV.moveInto(Result); } else if (T->isVectorType()) { if (!EvaluateVector(E, Result, Info)) return false; } else if (T->isIntegralOrEnumerationType()) { if (!IntExprEvaluator(Info, Result).Visit(E)) return false; } else if (T->hasPointerRepresentation()) { LValue LV; if (!EvaluatePointer(E, LV, Info)) return false; LV.moveInto(Result); } else if (T->isRealFloatingType()) { llvm::APFloat F(0.0); if (!EvaluateFloat(E, F, Info)) return false; Result = APValue(F); } else if (T->isAnyComplexType()) { ComplexValue C; if (!EvaluateComplex(E, C, Info)) return false; C.moveInto(Result); } else if (T->isFixedPointType()) { if (!FixedPointExprEvaluator(Info, Result).Visit(E)) return false; } else if (T->isMemberPointerType()) { MemberPtr P; if (!EvaluateMemberPointer(E, P, Info)) return false; P.moveInto(Result); return true; } else if (T->isArrayType()) { LValue LV; APValue &Value = createTemporary(E, false, LV, *Info.CurrentCall); if (!EvaluateArray(E, LV, Value, Info)) return false; Result = Value; } else if (T->isRecordType()) { LValue LV; APValue &Value = createTemporary(E, false, LV, *Info.CurrentCall); if (!EvaluateRecord(E, LV, Value, Info)) return false; Result = Value; } else if (T->isVoidType()) { if (!Info.getLangOpts().CPlusPlus11) Info.CCEDiag(E, diag::note_constexpr_nonliteral) << E->getType(); if (!EvaluateVoid(E, Info)) return false; } else if (T->isAtomicType()) { QualType Unqual = T.getAtomicUnqualifiedType(); if (Unqual->isArrayType() || Unqual->isRecordType()) { LValue LV; APValue &Value = createTemporary(E, false, LV, *Info.CurrentCall); if (!EvaluateAtomic(E, &LV, Value, Info)) return false; } else { if (!EvaluateAtomic(E, nullptr, Result, Info)) return false; } } else if (Info.getLangOpts().CPlusPlus11) { Info.FFDiag(E, diag::note_constexpr_nonliteral) << E->getType(); return false; } else { Info.FFDiag(E, diag::note_invalid_subexpr_in_const_expr); return false; } return true; } /// EvaluateInPlace - Evaluate an expression in-place in an APValue. In some /// cases, the in-place evaluation is essential, since later initializers for /// an object can indirectly refer to subobjects which were initialized earlier. static bool EvaluateInPlace(APValue &Result, EvalInfo &Info, const LValue &This, const Expr *E, bool AllowNonLiteralTypes) { assert(!E->isValueDependent()); if (!AllowNonLiteralTypes && !CheckLiteralType(Info, E, &This)) return false; if (E->isRValue()) { // Evaluate arrays and record types in-place, so that later initializers can // refer to earlier-initialized members of the object. QualType T = E->getType(); if (T->isArrayType()) return EvaluateArray(E, This, Result, Info); else if (T->isRecordType()) return EvaluateRecord(E, This, Result, Info); else if (T->isAtomicType()) { QualType Unqual = T.getAtomicUnqualifiedType(); if (Unqual->isArrayType() || Unqual->isRecordType()) return EvaluateAtomic(E, &This, Result, Info); } } // For any other type, in-place evaluation is unimportant. return Evaluate(Result, Info, E); } /// EvaluateAsRValue - Try to evaluate this expression, performing an implicit /// lvalue-to-rvalue cast if it is an lvalue. static bool EvaluateAsRValue(EvalInfo &Info, const Expr *E, APValue &Result) { if (E->getType().isNull()) return false; if (!CheckLiteralType(Info, E)) return false; if (!::Evaluate(Result, Info, E)) return false; if (E->isGLValue()) { LValue LV; LV.setFrom(Info.Ctx, Result); if (!handleLValueToRValueConversion(Info, E, E->getType(), LV, Result)) return false; } // Check this core constant expression is a constant expression. return CheckConstantExpression(Info, E->getExprLoc(), E->getType(), Result); } static bool FastEvaluateAsRValue(const Expr *Exp, Expr::EvalResult &Result, const ASTContext &Ctx, bool &IsConst) { // Fast-path evaluations of integer literals, since we sometimes see files // containing vast quantities of these. if (const IntegerLiteral *L = dyn_cast(Exp)) { Result.Val = APValue(APSInt(L->getValue(), L->getType()->isUnsignedIntegerType())); IsConst = true; return true; } // This case should be rare, but we need to check it before we check on // the type below. if (Exp->getType().isNull()) { IsConst = false; return true; } // FIXME: Evaluating values of large array and record types can cause // performance problems. Only do so in C++11 for now. if (Exp->isRValue() && (Exp->getType()->isArrayType() || Exp->getType()->isRecordType()) && !Ctx.getLangOpts().CPlusPlus11) { IsConst = false; return true; } return false; } static bool hasUnacceptableSideEffect(Expr::EvalStatus &Result, Expr::SideEffectsKind SEK) { return (SEK < Expr::SE_AllowSideEffects && Result.HasSideEffects) || (SEK < Expr::SE_AllowUndefinedBehavior && Result.HasUndefinedBehavior); } static bool EvaluateAsRValue(const Expr *E, Expr::EvalResult &Result, const ASTContext &Ctx, EvalInfo &Info) { bool IsConst; if (FastEvaluateAsRValue(E, Result, Ctx, IsConst)) return IsConst; return EvaluateAsRValue(Info, E, Result.Val); } static bool EvaluateAsInt(const Expr *E, Expr::EvalResult &ExprResult, const ASTContext &Ctx, Expr::SideEffectsKind AllowSideEffects, EvalInfo &Info) { if (!E->getType()->isIntegralOrEnumerationType()) return false; if (!::EvaluateAsRValue(E, ExprResult, Ctx, Info) || !ExprResult.Val.isInt() || hasUnacceptableSideEffect(ExprResult, AllowSideEffects)) return false; return true; } static bool EvaluateAsFixedPoint(const Expr *E, Expr::EvalResult &ExprResult, const ASTContext &Ctx, Expr::SideEffectsKind AllowSideEffects, EvalInfo &Info) { if (!E->getType()->isFixedPointType()) return false; if (!::EvaluateAsRValue(E, ExprResult, Ctx, Info)) return false; if (!ExprResult.Val.isFixedPoint() || hasUnacceptableSideEffect(ExprResult, AllowSideEffects)) return false; return true; } /// EvaluateAsRValue - Return true if this is a constant which we can fold using /// any crazy technique (that has nothing to do with language standards) that /// we want to. If this function returns true, it returns the folded constant /// in Result. If this expression is a glvalue, an lvalue-to-rvalue conversion /// will be applied to the result. bool Expr::EvaluateAsRValue(EvalResult &Result, const ASTContext &Ctx, bool InConstantContext) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); EvalInfo Info(Ctx, Result, EvalInfo::EM_IgnoreSideEffects); Info.InConstantContext = InConstantContext; return ::EvaluateAsRValue(this, Result, Ctx, Info); } bool Expr::EvaluateAsBooleanCondition(bool &Result, const ASTContext &Ctx, bool InConstantContext) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); EvalResult Scratch; return EvaluateAsRValue(Scratch, Ctx, InConstantContext) && HandleConversionToBool(Scratch.Val, Result); } bool Expr::EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects, bool InConstantContext) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); EvalInfo Info(Ctx, Result, EvalInfo::EM_IgnoreSideEffects); Info.InConstantContext = InConstantContext; return ::EvaluateAsInt(this, Result, Ctx, AllowSideEffects, Info); } bool Expr::EvaluateAsFixedPoint(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects, bool InConstantContext) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); EvalInfo Info(Ctx, Result, EvalInfo::EM_IgnoreSideEffects); Info.InConstantContext = InConstantContext; return ::EvaluateAsFixedPoint(this, Result, Ctx, AllowSideEffects, Info); } bool Expr::EvaluateAsFloat(APFloat &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects, bool InConstantContext) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); if (!getType()->isRealFloatingType()) return false; EvalResult ExprResult; if (!EvaluateAsRValue(ExprResult, Ctx, InConstantContext) || !ExprResult.Val.isFloat() || hasUnacceptableSideEffect(ExprResult, AllowSideEffects)) return false; Result = ExprResult.Val.getFloat(); return true; } bool Expr::EvaluateAsLValue(EvalResult &Result, const ASTContext &Ctx, bool InConstantContext) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); EvalInfo Info(Ctx, Result, EvalInfo::EM_ConstantFold); Info.InConstantContext = InConstantContext; LValue LV; if (!EvaluateLValue(this, LV, Info) || Result.HasSideEffects || !CheckLValueConstantExpression(Info, getExprLoc(), Ctx.getLValueReferenceType(getType()), LV, Expr::EvaluateForCodeGen)) return false; LV.moveInto(Result.Val); return true; } bool Expr::EvaluateAsConstantExpr(EvalResult &Result, ConstExprUsage Usage, const ASTContext &Ctx) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); EvalInfo::EvaluationMode EM = EvalInfo::EM_ConstantExpression; EvalInfo Info(Ctx, Result, EM); Info.InConstantContext = true; if (!::Evaluate(Result.Val, Info, this)) return false; return CheckConstantExpression(Info, getExprLoc(), getType(), Result.Val, Usage); } bool Expr::EvaluateAsInitializer(APValue &Value, const ASTContext &Ctx, const VarDecl *VD, SmallVectorImpl &Notes) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); // FIXME: Evaluating initializers for large array and record types can cause // performance problems. Only do so in C++11 for now. if (isRValue() && (getType()->isArrayType() || getType()->isRecordType()) && !Ctx.getLangOpts().CPlusPlus11) return false; Expr::EvalStatus EStatus; EStatus.Diag = &Notes; EvalInfo InitInfo(Ctx, EStatus, VD->isConstexpr() ? EvalInfo::EM_ConstantExpression : EvalInfo::EM_ConstantFold); InitInfo.setEvaluatingDecl(VD, Value); InitInfo.InConstantContext = true; LValue LVal; LVal.set(VD); // C++11 [basic.start.init]p2: // Variables with static storage duration or thread storage duration shall be // zero-initialized before any other initialization takes place. // This behavior is not present in C. if (Ctx.getLangOpts().CPlusPlus && !VD->hasLocalStorage() && !VD->getType()->isReferenceType()) { ImplicitValueInitExpr VIE(VD->getType()); if (!EvaluateInPlace(Value, InitInfo, LVal, &VIE, /*AllowNonLiteralTypes=*/true)) return false; } if (!EvaluateInPlace(Value, InitInfo, LVal, this, /*AllowNonLiteralTypes=*/true) || EStatus.HasSideEffects) return false; return CheckConstantExpression(InitInfo, VD->getLocation(), VD->getType(), Value); } /// isEvaluatable - Call EvaluateAsRValue to see if this expression can be /// constant folded, but discard the result. bool Expr::isEvaluatable(const ASTContext &Ctx, SideEffectsKind SEK) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); EvalResult Result; return EvaluateAsRValue(Result, Ctx, /* in constant context */ true) && !hasUnacceptableSideEffect(Result, SEK); } APSInt Expr::EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl *Diag) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); EvalResult EVResult; EVResult.Diag = Diag; EvalInfo Info(Ctx, EVResult, EvalInfo::EM_IgnoreSideEffects); Info.InConstantContext = true; bool Result = ::EvaluateAsRValue(this, EVResult, Ctx, Info); (void)Result; assert(Result && "Could not evaluate expression"); assert(EVResult.Val.isInt() && "Expression did not evaluate to integer"); return EVResult.Val.getInt(); } APSInt Expr::EvaluateKnownConstIntCheckOverflow( const ASTContext &Ctx, SmallVectorImpl *Diag) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); EvalResult EVResult; EVResult.Diag = Diag; EvalInfo Info(Ctx, EVResult, EvalInfo::EM_IgnoreSideEffects); Info.InConstantContext = true; Info.CheckingForUndefinedBehavior = true; bool Result = ::EvaluateAsRValue(Info, this, EVResult.Val); (void)Result; assert(Result && "Could not evaluate expression"); assert(EVResult.Val.isInt() && "Expression did not evaluate to integer"); return EVResult.Val.getInt(); } void Expr::EvaluateForOverflow(const ASTContext &Ctx) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); bool IsConst; EvalResult EVResult; if (!FastEvaluateAsRValue(this, EVResult, Ctx, IsConst)) { EvalInfo Info(Ctx, EVResult, EvalInfo::EM_IgnoreSideEffects); Info.CheckingForUndefinedBehavior = true; (void)::EvaluateAsRValue(Info, this, EVResult.Val); } } bool Expr::EvalResult::isGlobalLValue() const { assert(Val.isLValue()); return IsGlobalLValue(Val.getLValueBase()); } /// isIntegerConstantExpr - this recursive routine will test if an expression is /// an integer constant expression. /// FIXME: Pass up a reason why! Invalid operation in i-c-e, division by zero, /// comma, etc // CheckICE - This function does the fundamental ICE checking: the returned // ICEDiag contains an ICEKind indicating whether the expression is an ICE, // and a (possibly null) SourceLocation indicating the location of the problem. // // Note that to reduce code duplication, this helper does no evaluation // itself; the caller checks whether the expression is evaluatable, and // in the rare cases where CheckICE actually cares about the evaluated // value, it calls into Evaluate. namespace { enum ICEKind { /// This expression is an ICE. IK_ICE, /// This expression is not an ICE, but if it isn't evaluated, it's /// a legal subexpression for an ICE. This return value is used to handle /// the comma operator in C99 mode, and non-constant subexpressions. IK_ICEIfUnevaluated, /// This expression is not an ICE, and is not a legal subexpression for one. IK_NotICE }; struct ICEDiag { ICEKind Kind; SourceLocation Loc; ICEDiag(ICEKind IK, SourceLocation l) : Kind(IK), Loc(l) {} }; } static ICEDiag NoDiag() { return ICEDiag(IK_ICE, SourceLocation()); } static ICEDiag Worst(ICEDiag A, ICEDiag B) { return A.Kind >= B.Kind ? A : B; } static ICEDiag CheckEvalInICE(const Expr* E, const ASTContext &Ctx) { Expr::EvalResult EVResult; Expr::EvalStatus Status; EvalInfo Info(Ctx, Status, EvalInfo::EM_ConstantExpression); Info.InConstantContext = true; if (!::EvaluateAsRValue(E, EVResult, Ctx, Info) || EVResult.HasSideEffects || !EVResult.Val.isInt()) return ICEDiag(IK_NotICE, E->getBeginLoc()); return NoDiag(); } static ICEDiag CheckICE(const Expr* E, const ASTContext &Ctx) { assert(!E->isValueDependent() && "Should not see value dependent exprs!"); if (!E->getType()->isIntegralOrEnumerationType()) return ICEDiag(IK_NotICE, E->getBeginLoc()); switch (E->getStmtClass()) { #define ABSTRACT_STMT(Node) #define STMT(Node, Base) case Expr::Node##Class: #define EXPR(Node, Base) #include "clang/AST/StmtNodes.inc" case Expr::PredefinedExprClass: case Expr::FloatingLiteralClass: case Expr::ImaginaryLiteralClass: case Expr::StringLiteralClass: case Expr::ArraySubscriptExprClass: case Expr::OMPArraySectionExprClass: case Expr::MemberExprClass: case Expr::CompoundAssignOperatorClass: case Expr::CompoundLiteralExprClass: case Expr::ExtVectorElementExprClass: case Expr::DesignatedInitExprClass: case Expr::ArrayInitLoopExprClass: case Expr::ArrayInitIndexExprClass: case Expr::NoInitExprClass: case Expr::DesignatedInitUpdateExprClass: case Expr::ImplicitValueInitExprClass: case Expr::ParenListExprClass: case Expr::VAArgExprClass: case Expr::AddrLabelExprClass: case Expr::StmtExprClass: case Expr::CXXMemberCallExprClass: case Expr::CUDAKernelCallExprClass: case Expr::CXXDynamicCastExprClass: case Expr::CXXTypeidExprClass: case Expr::CXXUuidofExprClass: case Expr::MSPropertyRefExprClass: case Expr::MSPropertySubscriptExprClass: case Expr::CXXNullPtrLiteralExprClass: case Expr::UserDefinedLiteralClass: case Expr::CXXThisExprClass: case Expr::CXXThrowExprClass: case Expr::CXXNewExprClass: case Expr::CXXDeleteExprClass: case Expr::CXXPseudoDestructorExprClass: case Expr::UnresolvedLookupExprClass: case Expr::TypoExprClass: case Expr::DependentScopeDeclRefExprClass: case Expr::CXXConstructExprClass: case Expr::CXXInheritedCtorInitExprClass: case Expr::CXXStdInitializerListExprClass: case Expr::CXXBindTemporaryExprClass: case Expr::ExprWithCleanupsClass: case Expr::CXXTemporaryObjectExprClass: case Expr::CXXUnresolvedConstructExprClass: case Expr::CXXDependentScopeMemberExprClass: case Expr::UnresolvedMemberExprClass: case Expr::ObjCStringLiteralClass: case Expr::ObjCBoxedExprClass: case Expr::ObjCArrayLiteralClass: case Expr::ObjCDictionaryLiteralClass: case Expr::ObjCEncodeExprClass: case Expr::ObjCMessageExprClass: case Expr::ObjCSelectorExprClass: case Expr::ObjCProtocolExprClass: case Expr::ObjCIvarRefExprClass: case Expr::ObjCPropertyRefExprClass: case Expr::ObjCSubscriptRefExprClass: case Expr::ObjCIsaExprClass: case Expr::ObjCAvailabilityCheckExprClass: case Expr::ShuffleVectorExprClass: case Expr::ConvertVectorExprClass: case Expr::BlockExprClass: case Expr::NoStmtClass: case Expr::OpaqueValueExprClass: case Expr::PackExpansionExprClass: case Expr::SubstNonTypeTemplateParmPackExprClass: case Expr::FunctionParmPackExprClass: case Expr::AsTypeExprClass: case Expr::ObjCIndirectCopyRestoreExprClass: case Expr::MaterializeTemporaryExprClass: case Expr::PseudoObjectExprClass: case Expr::AtomicExprClass: case Expr::LambdaExprClass: case Expr::CXXFoldExprClass: case Expr::CoawaitExprClass: case Expr::DependentCoawaitExprClass: case Expr::CoyieldExprClass: return ICEDiag(IK_NotICE, E->getBeginLoc()); case Expr::InitListExprClass: { // C++03 [dcl.init]p13: If T is a scalar type, then a declaration of the // form "T x = { a };" is equivalent to "T x = a;". // Unless we're initializing a reference, T is a scalar as it is known to be // of integral or enumeration type. if (E->isRValue()) if (cast(E)->getNumInits() == 1) return CheckICE(cast(E)->getInit(0), Ctx); return ICEDiag(IK_NotICE, E->getBeginLoc()); } case Expr::SizeOfPackExprClass: case Expr::GNUNullExprClass: case Expr::SourceLocExprClass: return NoDiag(); case Expr::SubstNonTypeTemplateParmExprClass: return CheckICE(cast(E)->getReplacement(), Ctx); case Expr::ConstantExprClass: return CheckICE(cast(E)->getSubExpr(), Ctx); case Expr::ParenExprClass: return CheckICE(cast(E)->getSubExpr(), Ctx); case Expr::GenericSelectionExprClass: return CheckICE(cast(E)->getResultExpr(), Ctx); case Expr::IntegerLiteralClass: case Expr::FixedPointLiteralClass: case Expr::CharacterLiteralClass: case Expr::ObjCBoolLiteralExprClass: case Expr::CXXBoolLiteralExprClass: case Expr::CXXScalarValueInitExprClass: case Expr::TypeTraitExprClass: case Expr::ArrayTypeTraitExprClass: case Expr::ExpressionTraitExprClass: case Expr::CXXNoexceptExprClass: return NoDiag(); case Expr::CallExprClass: case Expr::CXXOperatorCallExprClass: { // C99 6.6/3 allows function calls within unevaluated subexpressions of // constant expressions, but they can never be ICEs because an ICE cannot // contain an operand of (pointer to) function type. const CallExpr *CE = cast(E); if (CE->getBuiltinCallee()) return CheckEvalInICE(E, Ctx); return ICEDiag(IK_NotICE, E->getBeginLoc()); } case Expr::DeclRefExprClass: { if (isa(cast(E)->getDecl())) return NoDiag(); const ValueDecl *D = cast(E)->getDecl(); if (Ctx.getLangOpts().CPlusPlus && D && IsConstNonVolatile(D->getType())) { // Parameter variables are never constants. Without this check, // getAnyInitializer() can find a default argument, which leads // to chaos. if (isa(D)) return ICEDiag(IK_NotICE, cast(E)->getLocation()); // C++ 7.1.5.1p2 // A variable of non-volatile const-qualified integral or enumeration // type initialized by an ICE can be used in ICEs. if (const VarDecl *Dcl = dyn_cast(D)) { if (!Dcl->getType()->isIntegralOrEnumerationType()) return ICEDiag(IK_NotICE, cast(E)->getLocation()); const VarDecl *VD; // Look for a declaration of this variable that has an initializer, and // check whether it is an ICE. if (Dcl->getAnyInitializer(VD) && VD->checkInitIsICE()) return NoDiag(); else return ICEDiag(IK_NotICE, cast(E)->getLocation()); } } return ICEDiag(IK_NotICE, E->getBeginLoc()); } case Expr::UnaryOperatorClass: { const UnaryOperator *Exp = cast(E); switch (Exp->getOpcode()) { case UO_PostInc: case UO_PostDec: case UO_PreInc: case UO_PreDec: case UO_AddrOf: case UO_Deref: case UO_Coawait: // C99 6.6/3 allows increment and decrement within unevaluated // subexpressions of constant expressions, but they can never be ICEs // because an ICE cannot contain an lvalue operand. return ICEDiag(IK_NotICE, E->getBeginLoc()); case UO_Extension: case UO_LNot: case UO_Plus: case UO_Minus: case UO_Not: case UO_Real: case UO_Imag: return CheckICE(Exp->getSubExpr(), Ctx); } llvm_unreachable("invalid unary operator class"); } case Expr::OffsetOfExprClass: { // Note that per C99, offsetof must be an ICE. And AFAIK, using // EvaluateAsRValue matches the proposed gcc behavior for cases like // "offsetof(struct s{int x[4];}, x[1.0])". This doesn't affect // compliance: we should warn earlier for offsetof expressions with // array subscripts that aren't ICEs, and if the array subscripts // are ICEs, the value of the offsetof must be an integer constant. return CheckEvalInICE(E, Ctx); } case Expr::UnaryExprOrTypeTraitExprClass: { const UnaryExprOrTypeTraitExpr *Exp = cast(E); if ((Exp->getKind() == UETT_SizeOf) && Exp->getTypeOfArgument()->isVariableArrayType()) return ICEDiag(IK_NotICE, E->getBeginLoc()); return NoDiag(); } case Expr::BinaryOperatorClass: { const BinaryOperator *Exp = cast(E); switch (Exp->getOpcode()) { case BO_PtrMemD: case BO_PtrMemI: case BO_Assign: case BO_MulAssign: case BO_DivAssign: case BO_RemAssign: case BO_AddAssign: case BO_SubAssign: case BO_ShlAssign: case BO_ShrAssign: case BO_AndAssign: case BO_XorAssign: case BO_OrAssign: // C99 6.6/3 allows assignments within unevaluated subexpressions of // constant expressions, but they can never be ICEs because an ICE cannot // contain an lvalue operand. return ICEDiag(IK_NotICE, E->getBeginLoc()); case BO_Mul: case BO_Div: case BO_Rem: case BO_Add: case BO_Sub: case BO_Shl: case BO_Shr: case BO_LT: case BO_GT: case BO_LE: case BO_GE: case BO_EQ: case BO_NE: case BO_And: case BO_Xor: case BO_Or: case BO_Comma: case BO_Cmp: { ICEDiag LHSResult = CheckICE(Exp->getLHS(), Ctx); ICEDiag RHSResult = CheckICE(Exp->getRHS(), Ctx); if (Exp->getOpcode() == BO_Div || Exp->getOpcode() == BO_Rem) { // EvaluateAsRValue gives an error for undefined Div/Rem, so make sure // we don't evaluate one. if (LHSResult.Kind == IK_ICE && RHSResult.Kind == IK_ICE) { llvm::APSInt REval = Exp->getRHS()->EvaluateKnownConstInt(Ctx); if (REval == 0) return ICEDiag(IK_ICEIfUnevaluated, E->getBeginLoc()); if (REval.isSigned() && REval.isAllOnesValue()) { llvm::APSInt LEval = Exp->getLHS()->EvaluateKnownConstInt(Ctx); if (LEval.isMinSignedValue()) return ICEDiag(IK_ICEIfUnevaluated, E->getBeginLoc()); } } } if (Exp->getOpcode() == BO_Comma) { if (Ctx.getLangOpts().C99) { // C99 6.6p3 introduces a strange edge case: comma can be in an ICE // if it isn't evaluated. if (LHSResult.Kind == IK_ICE && RHSResult.Kind == IK_ICE) return ICEDiag(IK_ICEIfUnevaluated, E->getBeginLoc()); } else { // In both C89 and C++, commas in ICEs are illegal. return ICEDiag(IK_NotICE, E->getBeginLoc()); } } return Worst(LHSResult, RHSResult); } case BO_LAnd: case BO_LOr: { ICEDiag LHSResult = CheckICE(Exp->getLHS(), Ctx); ICEDiag RHSResult = CheckICE(Exp->getRHS(), Ctx); if (LHSResult.Kind == IK_ICE && RHSResult.Kind == IK_ICEIfUnevaluated) { // Rare case where the RHS has a comma "side-effect"; we need // to actually check the condition to see whether the side // with the comma is evaluated. if ((Exp->getOpcode() == BO_LAnd) != (Exp->getLHS()->EvaluateKnownConstInt(Ctx) == 0)) return RHSResult; return NoDiag(); } return Worst(LHSResult, RHSResult); } } llvm_unreachable("invalid binary operator kind"); } case Expr::ImplicitCastExprClass: case Expr::CStyleCastExprClass: case Expr::CXXFunctionalCastExprClass: case Expr::CXXStaticCastExprClass: case Expr::CXXReinterpretCastExprClass: case Expr::CXXConstCastExprClass: case Expr::ObjCBridgedCastExprClass: { const Expr *SubExpr = cast(E)->getSubExpr(); if (isa(E)) { if (const FloatingLiteral *FL = dyn_cast(SubExpr->IgnoreParenImpCasts())) { unsigned DestWidth = Ctx.getIntWidth(E->getType()); bool DestSigned = E->getType()->isSignedIntegerOrEnumerationType(); APSInt IgnoredVal(DestWidth, !DestSigned); bool Ignored; // If the value does not fit in the destination type, the behavior is // undefined, so we are not required to treat it as a constant // expression. if (FL->getValue().convertToInteger(IgnoredVal, llvm::APFloat::rmTowardZero, &Ignored) & APFloat::opInvalidOp) return ICEDiag(IK_NotICE, E->getBeginLoc()); return NoDiag(); } } switch (cast(E)->getCastKind()) { case CK_LValueToRValue: case CK_AtomicToNonAtomic: case CK_NonAtomicToAtomic: case CK_NoOp: case CK_IntegralToBoolean: case CK_IntegralCast: return CheckICE(SubExpr, Ctx); default: return ICEDiag(IK_NotICE, E->getBeginLoc()); } } case Expr::BinaryConditionalOperatorClass: { const BinaryConditionalOperator *Exp = cast(E); ICEDiag CommonResult = CheckICE(Exp->getCommon(), Ctx); if (CommonResult.Kind == IK_NotICE) return CommonResult; ICEDiag FalseResult = CheckICE(Exp->getFalseExpr(), Ctx); if (FalseResult.Kind == IK_NotICE) return FalseResult; if (CommonResult.Kind == IK_ICEIfUnevaluated) return CommonResult; if (FalseResult.Kind == IK_ICEIfUnevaluated && Exp->getCommon()->EvaluateKnownConstInt(Ctx) != 0) return NoDiag(); return FalseResult; } case Expr::ConditionalOperatorClass: { const ConditionalOperator *Exp = cast(E); // If the condition (ignoring parens) is a __builtin_constant_p call, // then only the true side is actually considered in an integer constant // expression, and it is fully evaluated. This is an important GNU // extension. See GCC PR38377 for discussion. if (const CallExpr *CallCE = dyn_cast(Exp->getCond()->IgnoreParenCasts())) if (CallCE->getBuiltinCallee() == Builtin::BI__builtin_constant_p) return CheckEvalInICE(E, Ctx); ICEDiag CondResult = CheckICE(Exp->getCond(), Ctx); if (CondResult.Kind == IK_NotICE) return CondResult; ICEDiag TrueResult = CheckICE(Exp->getTrueExpr(), Ctx); ICEDiag FalseResult = CheckICE(Exp->getFalseExpr(), Ctx); if (TrueResult.Kind == IK_NotICE) return TrueResult; if (FalseResult.Kind == IK_NotICE) return FalseResult; if (CondResult.Kind == IK_ICEIfUnevaluated) return CondResult; if (TrueResult.Kind == IK_ICE && FalseResult.Kind == IK_ICE) return NoDiag(); // Rare case where the diagnostics depend on which side is evaluated // Note that if we get here, CondResult is 0, and at least one of // TrueResult and FalseResult is non-zero. if (Exp->getCond()->EvaluateKnownConstInt(Ctx) == 0) return FalseResult; return TrueResult; } case Expr::CXXDefaultArgExprClass: return CheckICE(cast(E)->getExpr(), Ctx); case Expr::CXXDefaultInitExprClass: return CheckICE(cast(E)->getExpr(), Ctx); case Expr::ChooseExprClass: { return CheckICE(cast(E)->getChosenSubExpr(), Ctx); } case Expr::BuiltinBitCastExprClass: { if (!checkBitCastConstexprEligibility(nullptr, Ctx, cast(E))) return ICEDiag(IK_NotICE, E->getBeginLoc()); return CheckICE(cast(E)->getSubExpr(), Ctx); } } llvm_unreachable("Invalid StmtClass!"); } /// Evaluate an expression as a C++11 integral constant expression. static bool EvaluateCPlusPlus11IntegralConstantExpr(const ASTContext &Ctx, const Expr *E, llvm::APSInt *Value, SourceLocation *Loc) { if (!E->getType()->isIntegralOrUnscopedEnumerationType()) { if (Loc) *Loc = E->getExprLoc(); return false; } APValue Result; if (!E->isCXX11ConstantExpr(Ctx, &Result, Loc)) return false; if (!Result.isInt()) { if (Loc) *Loc = E->getExprLoc(); return false; } if (Value) *Value = Result.getInt(); return true; } bool Expr::isIntegerConstantExpr(const ASTContext &Ctx, SourceLocation *Loc) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); if (Ctx.getLangOpts().CPlusPlus11) return EvaluateCPlusPlus11IntegralConstantExpr(Ctx, this, nullptr, Loc); ICEDiag D = CheckICE(this, Ctx); if (D.Kind != IK_ICE) { if (Loc) *Loc = D.Loc; return false; } return true; } bool Expr::isIntegerConstantExpr(llvm::APSInt &Value, const ASTContext &Ctx, SourceLocation *Loc, bool isEvaluated) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); if (Ctx.getLangOpts().CPlusPlus11) return EvaluateCPlusPlus11IntegralConstantExpr(Ctx, this, &Value, Loc); if (!isIntegerConstantExpr(Ctx, Loc)) return false; // The only possible side-effects here are due to UB discovered in the // evaluation (for instance, INT_MAX + 1). In such a case, we are still // required to treat the expression as an ICE, so we produce the folded // value. EvalResult ExprResult; Expr::EvalStatus Status; EvalInfo Info(Ctx, Status, EvalInfo::EM_IgnoreSideEffects); Info.InConstantContext = true; if (!::EvaluateAsInt(this, ExprResult, Ctx, SE_AllowSideEffects, Info)) llvm_unreachable("ICE cannot be evaluated!"); Value = ExprResult.Val.getInt(); return true; } bool Expr::isCXX98IntegralConstantExpr(const ASTContext &Ctx) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); return CheckICE(this, Ctx).Kind == IK_ICE; } bool Expr::isCXX11ConstantExpr(const ASTContext &Ctx, APValue *Result, SourceLocation *Loc) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); // We support this checking in C++98 mode in order to diagnose compatibility // issues. assert(Ctx.getLangOpts().CPlusPlus); // Build evaluation settings. Expr::EvalStatus Status; SmallVector Diags; Status.Diag = &Diags; EvalInfo Info(Ctx, Status, EvalInfo::EM_ConstantExpression); APValue Scratch; bool IsConstExpr = ::EvaluateAsRValue(Info, this, Result ? *Result : Scratch); if (!Diags.empty()) { IsConstExpr = false; if (Loc) *Loc = Diags[0].first; } else if (!IsConstExpr) { // FIXME: This shouldn't happen. if (Loc) *Loc = getExprLoc(); } return IsConstExpr; } bool Expr::EvaluateWithSubstitution(APValue &Value, ASTContext &Ctx, const FunctionDecl *Callee, ArrayRef Args, const Expr *This) const { assert(!isValueDependent() && "Expression evaluator can't be called on a dependent expression."); Expr::EvalStatus Status; EvalInfo Info(Ctx, Status, EvalInfo::EM_ConstantExpressionUnevaluated); Info.InConstantContext = true; LValue ThisVal; const LValue *ThisPtr = nullptr; if (This) { #ifndef NDEBUG auto *MD = dyn_cast(Callee); assert(MD && "Don't provide `this` for non-methods."); assert(!MD->isStatic() && "Don't provide `this` for static methods."); #endif if (EvaluateObjectArgument(Info, This, ThisVal)) ThisPtr = &ThisVal; if (Info.EvalStatus.HasSideEffects) return false; } ArgVector ArgValues(Args.size()); for (ArrayRef::iterator I = Args.begin(), E = Args.end(); I != E; ++I) { if ((*I)->isValueDependent() || !Evaluate(ArgValues[I - Args.begin()], Info, *I)) // If evaluation fails, throw away the argument entirely. ArgValues[I - Args.begin()] = APValue(); if (Info.EvalStatus.HasSideEffects) return false; } // Build fake call to Callee. CallStackFrame Frame(Info, Callee->getLocation(), Callee, ThisPtr, ArgValues.data()); return Evaluate(Value, Info, this) && !Info.EvalStatus.HasSideEffects; } bool Expr::isPotentialConstantExpr(const FunctionDecl *FD, SmallVectorImpl< PartialDiagnosticAt> &Diags) { // FIXME: It would be useful to check constexpr function templates, but at the // moment the constant expression evaluator cannot cope with the non-rigorous // ASTs which we build for dependent expressions. if (FD->isDependentContext()) return true; Expr::EvalStatus Status; Status.Diag = &Diags; EvalInfo Info(FD->getASTContext(), Status, EvalInfo::EM_ConstantExpression); Info.InConstantContext = true; Info.CheckingPotentialConstantExpression = true; const CXXMethodDecl *MD = dyn_cast(FD); const CXXRecordDecl *RD = MD ? MD->getParent()->getCanonicalDecl() : nullptr; // Fabricate an arbitrary expression on the stack and pretend that it // is a temporary being used as the 'this' pointer. LValue This; ImplicitValueInitExpr VIE(RD ? Info.Ctx.getRecordType(RD) : Info.Ctx.IntTy); This.set({&VIE, Info.CurrentCall->Index}); ArrayRef Args; APValue Scratch; if (const CXXConstructorDecl *CD = dyn_cast(FD)) { // Evaluate the call as a constant initializer, to allow the construction // of objects of non-literal types. Info.setEvaluatingDecl(This.getLValueBase(), Scratch); HandleConstructorCall(&VIE, This, Args, CD, Info, Scratch); } else { SourceLocation Loc = FD->getLocation(); HandleFunctionCall(Loc, FD, (MD && MD->isInstance()) ? &This : nullptr, Args, FD->getBody(), Info, Scratch, nullptr); } return Diags.empty(); } bool Expr::isPotentialConstantExprUnevaluated(Expr *E, const FunctionDecl *FD, SmallVectorImpl< PartialDiagnosticAt> &Diags) { assert(!E->isValueDependent() && "Expression evaluator can't be called on a dependent expression."); Expr::EvalStatus Status; Status.Diag = &Diags; EvalInfo Info(FD->getASTContext(), Status, EvalInfo::EM_ConstantExpressionUnevaluated); Info.InConstantContext = true; Info.CheckingPotentialConstantExpression = true; // Fabricate a call stack frame to give the arguments a plausible cover story. ArrayRef Args; ArgVector ArgValues(0); bool Success = EvaluateArgs(Args, ArgValues, Info, FD); (void)Success; assert(Success && "Failed to set up arguments for potential constant evaluation"); CallStackFrame Frame(Info, SourceLocation(), FD, nullptr, ArgValues.data()); APValue ResultScratch; Evaluate(ResultScratch, Info, E); return Diags.empty(); } bool Expr::tryEvaluateObjectSize(uint64_t &Result, ASTContext &Ctx, unsigned Type) const { if (!getType()->isPointerType()) return false; Expr::EvalStatus Status; EvalInfo Info(Ctx, Status, EvalInfo::EM_ConstantFold); return tryEvaluateBuiltinObjectSize(this, Type, Info, Result); } Index: stable/12/contrib/llvm-project/clang/lib/Basic/Targets/PPC.h =================================================================== --- stable/12/contrib/llvm-project/clang/lib/Basic/Targets/PPC.h (revision 356466) +++ stable/12/contrib/llvm-project/clang/lib/Basic/Targets/PPC.h (revision 356467) @@ -1,466 +1,483 @@ //===--- PPC.h - Declare PPC target feature support -------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file declares PPC TargetInfo objects. // //===----------------------------------------------------------------------===// #ifndef LLVM_CLANG_LIB_BASIC_TARGETS_PPC_H #define LLVM_CLANG_LIB_BASIC_TARGETS_PPC_H #include "OSTargets.h" #include "clang/Basic/TargetInfo.h" #include "clang/Basic/TargetOptions.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Support/Compiler.h" namespace clang { namespace targets { // PPC abstract base class class LLVM_LIBRARY_VISIBILITY PPCTargetInfo : public TargetInfo { /// Flags for architecture specific defines. typedef enum { ArchDefineNone = 0, ArchDefineName = 1 << 0, // is substituted for arch name. ArchDefinePpcgr = 1 << 1, ArchDefinePpcsq = 1 << 2, ArchDefine440 = 1 << 3, ArchDefine603 = 1 << 4, ArchDefine604 = 1 << 5, ArchDefinePwr4 = 1 << 6, ArchDefinePwr5 = 1 << 7, ArchDefinePwr5x = 1 << 8, ArchDefinePwr6 = 1 << 9, ArchDefinePwr6x = 1 << 10, ArchDefinePwr7 = 1 << 11, ArchDefinePwr8 = 1 << 12, ArchDefinePwr9 = 1 << 13, ArchDefineA2 = 1 << 14, ArchDefineA2q = 1 << 15 } ArchDefineTypes; ArchDefineTypes ArchDefs = ArchDefineNone; static const Builtin::Info BuiltinInfo[]; static const char *const GCCRegNames[]; static const TargetInfo::GCCRegAlias GCCRegAliases[]; std::string CPU; enum PPCFloatABI { HardFloat, SoftFloat } FloatABI; // Target cpu features. bool HasAltivec = false; bool HasVSX = false; bool HasP8Vector = false; bool HasP8Crypto = false; bool HasDirectMove = false; bool HasQPX = false; bool HasHTM = false; bool HasBPERMD = false; bool HasExtDiv = false; bool HasP9Vector = false; bool HasSPE = false; protected: std::string ABI; public: PPCTargetInfo(const llvm::Triple &Triple, const TargetOptions &) : TargetInfo(Triple) { SuitableAlign = 128; SimdDefaultAlign = 128; LongDoubleWidth = LongDoubleAlign = 128; LongDoubleFormat = &llvm::APFloat::PPCDoubleDouble(); } // Set the language option for altivec based on our value. void adjust(LangOptions &Opts) override; // Note: GCC recognizes the following additional cpus: // 401, 403, 405, 405fp, 440fp, 464, 464fp, 476, 476fp, 505, 740, 801, // 821, 823, 8540, 8548, e300c2, e300c3, e500mc64, e6500, 860, cell, // titan, rs64. bool isValidCPUName(StringRef Name) const override; void fillValidCPUList(SmallVectorImpl &Values) const override; bool setCPU(const std::string &Name) override { bool CPUKnown = isValidCPUName(Name); if (CPUKnown) { CPU = Name; // CPU identification. ArchDefs = (ArchDefineTypes)llvm::StringSwitch(CPU) .Case("440", ArchDefineName) .Case("450", ArchDefineName | ArchDefine440) .Case("601", ArchDefineName) .Case("602", ArchDefineName | ArchDefinePpcgr) .Case("603", ArchDefineName | ArchDefinePpcgr) .Case("603e", ArchDefineName | ArchDefine603 | ArchDefinePpcgr) .Case("603ev", ArchDefineName | ArchDefine603 | ArchDefinePpcgr) .Case("604", ArchDefineName | ArchDefinePpcgr) .Case("604e", ArchDefineName | ArchDefine604 | ArchDefinePpcgr) .Case("620", ArchDefineName | ArchDefinePpcgr) .Case("630", ArchDefineName | ArchDefinePpcgr) .Case("7400", ArchDefineName | ArchDefinePpcgr) .Case("7450", ArchDefineName | ArchDefinePpcgr) .Case("750", ArchDefineName | ArchDefinePpcgr) .Case("970", ArchDefineName | ArchDefinePwr4 | ArchDefinePpcgr | ArchDefinePpcsq) .Case("a2", ArchDefineA2) .Case("a2q", ArchDefineName | ArchDefineA2 | ArchDefineA2q) .Cases("power3", "pwr3", ArchDefinePpcgr) .Cases("power4", "pwr4", ArchDefinePwr4 | ArchDefinePpcgr | ArchDefinePpcsq) .Cases("power5", "pwr5", ArchDefinePwr5 | ArchDefinePwr4 | ArchDefinePpcgr | ArchDefinePpcsq) .Cases("power5x", "pwr5x", ArchDefinePwr5x | ArchDefinePwr5 | ArchDefinePwr4 | ArchDefinePpcgr | ArchDefinePpcsq) .Cases("power6", "pwr6", ArchDefinePwr6 | ArchDefinePwr5x | ArchDefinePwr5 | ArchDefinePwr4 | ArchDefinePpcgr | ArchDefinePpcsq) .Cases("power6x", "pwr6x", ArchDefinePwr6x | ArchDefinePwr6 | ArchDefinePwr5x | ArchDefinePwr5 | ArchDefinePwr4 | ArchDefinePpcgr | ArchDefinePpcsq) .Cases("power7", "pwr7", ArchDefinePwr7 | ArchDefinePwr6 | ArchDefinePwr5x | ArchDefinePwr5 | ArchDefinePwr4 | ArchDefinePpcgr | ArchDefinePpcsq) // powerpc64le automatically defaults to at least power8. .Cases("power8", "pwr8", "ppc64le", ArchDefinePwr8 | ArchDefinePwr7 | ArchDefinePwr6 | ArchDefinePwr5x | ArchDefinePwr5 | ArchDefinePwr4 | ArchDefinePpcgr | ArchDefinePpcsq) .Cases("power9", "pwr9", ArchDefinePwr9 | ArchDefinePwr8 | ArchDefinePwr7 | ArchDefinePwr6 | ArchDefinePwr5x | ArchDefinePwr5 | ArchDefinePwr4 | ArchDefinePpcgr | ArchDefinePpcsq) .Default(ArchDefineNone); } return CPUKnown; } StringRef getABI() const override { return ABI; } ArrayRef getTargetBuiltins() const override; bool isCLZForZeroUndef() const override { return false; } void getTargetDefines(const LangOptions &Opts, MacroBuilder &Builder) const override; bool initFeatureMap(llvm::StringMap &Features, DiagnosticsEngine &Diags, StringRef CPU, const std::vector &FeaturesVec) const override; bool handleTargetFeatures(std::vector &Features, DiagnosticsEngine &Diags) override; bool hasFeature(StringRef Feature) const override; void setFeatureEnabled(llvm::StringMap &Features, StringRef Name, bool Enabled) const override; ArrayRef getGCCRegNames() const override; ArrayRef getGCCRegAliases() const override; ArrayRef getGCCAddlRegNames() const override; bool validateAsmConstraint(const char *&Name, TargetInfo::ConstraintInfo &Info) const override { switch (*Name) { default: return false; case 'O': // Zero break; case 'f': // Floating point register // Don't use floating point registers on soft float ABI. if (FloatABI == SoftFloat) return false; LLVM_FALLTHROUGH; case 'b': // Base register Info.setAllowsRegister(); break; // FIXME: The following are added to allow parsing. // I just took a guess at what the actions should be. // Also, is more specific checking needed? I.e. specific registers? case 'd': // Floating point register (containing 64-bit value) case 'v': // Altivec vector register // Don't use floating point and altivec vector registers // on soft float ABI if (FloatABI == SoftFloat) return false; Info.setAllowsRegister(); break; case 'w': switch (Name[1]) { case 'd': // VSX vector register to hold vector double data case 'f': // VSX vector register to hold vector float data case 's': // VSX vector register to hold scalar double data case 'w': // VSX vector register to hold scalar double data case 'a': // Any VSX register case 'c': // An individual CR bit case 'i': // FP or VSX register to hold 64-bit integers data break; default: return false; } Info.setAllowsRegister(); Name++; // Skip over 'w'. break; case 'h': // `MQ', `CTR', or `LINK' register case 'q': // `MQ' register case 'c': // `CTR' register case 'l': // `LINK' register case 'x': // `CR' register (condition register) number 0 case 'y': // `CR' register (condition register) case 'z': // `XER[CA]' carry bit (part of the XER register) Info.setAllowsRegister(); break; case 'I': // Signed 16-bit constant case 'J': // Unsigned 16-bit constant shifted left 16 bits // (use `L' instead for SImode constants) case 'K': // Unsigned 16-bit constant case 'L': // Signed 16-bit constant shifted left 16 bits case 'M': // Constant larger than 31 case 'N': // Exact power of 2 case 'P': // Constant whose negation is a signed 16-bit constant case 'G': // Floating point constant that can be loaded into a // register with one instruction per word case 'H': // Integer/Floating point constant that can be loaded // into a register using three instructions break; case 'm': // Memory operand. Note that on PowerPC targets, m can // include addresses that update the base register. It // is therefore only safe to use `m' in an asm statement // if that asm statement accesses the operand exactly once. // The asm statement must also use `%U' as a // placeholder for the "update" flag in the corresponding // load or store instruction. For example: // asm ("st%U0 %1,%0" : "=m" (mem) : "r" (val)); // is correct but: // asm ("st %1,%0" : "=m" (mem) : "r" (val)); // is not. Use es rather than m if you don't want the base // register to be updated. case 'e': if (Name[1] != 's') return false; // es: A "stable" memory operand; that is, one which does not // include any automodification of the base register. Unlike // `m', this constraint can be used in asm statements that // might access the operand several times, or that might not // access it at all. Info.setAllowsMemory(); Name++; // Skip over 'e'. break; case 'Q': // Memory operand that is an offset from a register (it is // usually better to use `m' or `es' in asm statements) case 'Z': // Memory operand that is an indexed or indirect from a // register (it is usually better to use `m' or `es' in // asm statements) Info.setAllowsMemory(); Info.setAllowsRegister(); break; case 'R': // AIX TOC entry case 'a': // Address operand that is an indexed or indirect from a // register (`p' is preferable for asm statements) case 'S': // Constant suitable as a 64-bit mask operand case 'T': // Constant suitable as a 32-bit mask operand case 'U': // System V Release 4 small data area reference case 't': // AND masks that can be performed by two rldic{l, r} // instructions case 'W': // Vector constant that does not require memory case 'j': // Vector constant that is all zeros. break; // End FIXME. } return true; } std::string convertConstraint(const char *&Constraint) const override { std::string R; switch (*Constraint) { case 'e': case 'w': // Two-character constraint; add "^" hint for later parsing. R = std::string("^") + std::string(Constraint, 2); Constraint++; break; default: return TargetInfo::convertConstraint(Constraint); } return R; } const char *getClobbers() const override { return ""; } int getEHDataRegisterNumber(unsigned RegNo) const override { if (RegNo == 0) return 3; if (RegNo == 1) return 4; return -1; } bool hasSjLjLowering() const override { return true; } const char *getLongDoubleMangling() const override { if (LongDoubleWidth == 64) return "e"; return LongDoubleFormat == &llvm::APFloat::PPCDoubleDouble() ? "g" : "u9__ieee128"; } const char *getFloat128Mangling() const override { return "u9__ieee128"; } }; class LLVM_LIBRARY_VISIBILITY PPC32TargetInfo : public PPCTargetInfo { public: PPC32TargetInfo(const llvm::Triple &Triple, const TargetOptions &Opts) : PPCTargetInfo(Triple, Opts) { resetDataLayout("E-m:e-p:32:32-i64:64-n32"); switch (getTriple().getOS()) { case llvm::Triple::Linux: case llvm::Triple::FreeBSD: case llvm::Triple::NetBSD: SizeType = UnsignedInt; PtrDiffType = SignedInt; IntPtrType = SignedInt; break; case llvm::Triple::AIX: SizeType = UnsignedLong; PtrDiffType = SignedLong; IntPtrType = SignedLong; SuitableAlign = 64; break; default: break; } if (Triple.isOSFreeBSD() || Triple.isOSNetBSD() || Triple.isOSOpenBSD() || Triple.getOS() == llvm::Triple::AIX || Triple.isMusl()) { LongDoubleWidth = LongDoubleAlign = 64; LongDoubleFormat = &llvm::APFloat::IEEEdouble(); } // PPC32 supports atomics up to 4 bytes. MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 32; } BuiltinVaListKind getBuiltinVaListKind() const override { // This is the ELF definition, and is overridden by the Darwin sub-target return TargetInfo::PowerABIBuiltinVaList; } }; // Note: ABI differences may eventually require us to have a separate // TargetInfo for little endian. class LLVM_LIBRARY_VISIBILITY PPC64TargetInfo : public PPCTargetInfo { public: PPC64TargetInfo(const llvm::Triple &Triple, const TargetOptions &Opts) : PPCTargetInfo(Triple, Opts) { LongWidth = LongAlign = PointerWidth = PointerAlign = 64; IntMaxType = SignedLong; Int64Type = SignedLong; + if (Triple.getEnvironment() != llvm::Triple::UnknownEnvironment) { + switch (Triple.getEnvironment()){ + case llvm::Triple::ELFv1: + ABI = "elfv1"; + break; + default: + ABI = "elfv2"; + break; + } + } else { + if ((Triple.getOS() == llvm::Triple::FreeBSD) && + (Triple.getOSMajorVersion() < 13)) { + ABI = "elfv1"; + } else { + ABI = "elfv2"; + } + } + + if ((Triple.getArch() == llvm::Triple::ppc64le)) { resetDataLayout("e-m:e-i64:64-n32:64"); - ABI = "elfv2"; } else { resetDataLayout("E-m:e-i64:64-n32:64"); - ABI = Triple.getEnvironment() == llvm::Triple::ELFv2 ? "elfv2" : "elfv1"; } if (Triple.getOS() == llvm::Triple::AIX) SuitableAlign = 64; if (Triple.isOSFreeBSD() || Triple.getOS() == llvm::Triple::AIX || Triple.isMusl()) { LongDoubleWidth = LongDoubleAlign = 64; LongDoubleFormat = &llvm::APFloat::IEEEdouble(); } // PPC64 supports atomics up to 8 bytes. MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 64; } BuiltinVaListKind getBuiltinVaListKind() const override { return TargetInfo::CharPtrBuiltinVaList; } // PPC64 Linux-specific ABI options. bool setABI(const std::string &Name) override { if (Name == "elfv1" || Name == "elfv1-qpx" || Name == "elfv2") { ABI = Name; return true; } return false; } CallingConvCheckResult checkCallingConvention(CallingConv CC) const override { switch (CC) { case CC_Swift: return CCCR_OK; default: return CCCR_Warning; } } }; class LLVM_LIBRARY_VISIBILITY DarwinPPC32TargetInfo : public DarwinTargetInfo { public: DarwinPPC32TargetInfo(const llvm::Triple &Triple, const TargetOptions &Opts) : DarwinTargetInfo(Triple, Opts) { HasAlignMac68kSupport = true; BoolWidth = BoolAlign = 32; // XXX support -mone-byte-bool? PtrDiffType = SignedInt; // for http://llvm.org/bugs/show_bug.cgi?id=15726 LongLongAlign = 32; resetDataLayout("E-m:o-p:32:32-f64:32:64-n32"); } BuiltinVaListKind getBuiltinVaListKind() const override { return TargetInfo::CharPtrBuiltinVaList; } }; class LLVM_LIBRARY_VISIBILITY DarwinPPC64TargetInfo : public DarwinTargetInfo { public: DarwinPPC64TargetInfo(const llvm::Triple &Triple, const TargetOptions &Opts) : DarwinTargetInfo(Triple, Opts) { HasAlignMac68kSupport = true; resetDataLayout("E-m:o-i64:64-n32:64"); } }; class LLVM_LIBRARY_VISIBILITY AIXPPC32TargetInfo : public AIXTargetInfo { public: using AIXTargetInfo::AIXTargetInfo; BuiltinVaListKind getBuiltinVaListKind() const override { return TargetInfo::CharPtrBuiltinVaList; } }; class LLVM_LIBRARY_VISIBILITY AIXPPC64TargetInfo : public AIXTargetInfo { public: using AIXTargetInfo::AIXTargetInfo; }; } // namespace targets } // namespace clang #endif // LLVM_CLANG_LIB_BASIC_TARGETS_PPC_H Index: stable/12/contrib/llvm-project/clang =================================================================== --- stable/12/contrib/llvm-project/clang (revision 356466) +++ stable/12/contrib/llvm-project/clang (revision 356467) Property changes on: stable/12/contrib/llvm-project/clang ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/contrib/llvm-project/clang:r356104,356112 Index: stable/12/contrib/llvm-project/compiler-rt/lib/builtins/atomic.c =================================================================== --- stable/12/contrib/llvm-project/compiler-rt/lib/builtins/atomic.c (revision 356466) +++ stable/12/contrib/llvm-project/compiler-rt/lib/builtins/atomic.c (revision 356467) @@ -1,342 +1,349 @@ //===-- atomic.c - Implement support functions for atomic operations.------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // atomic.c defines a set of functions for performing atomic accesses on // arbitrary-sized memory locations. This design uses locks that should // be fast in the uncontended case, for two reasons: // // 1) This code must work with C programs that do not link to anything // (including pthreads) and so it should not depend on any pthread // functions. // 2) Atomic operations, rather than explicit mutexes, are most commonly used // on code where contended operations are rate. // // To avoid needing a per-object lock, this code allocates an array of // locks and hashes the object pointers to find the one that it should use. // For operations that must be atomic on two locations, the lower lock is // always acquired first, to avoid deadlock. // //===----------------------------------------------------------------------===// #include #include #include "assembly.h" // Clang objects if you redefine a builtin. This little hack allows us to // define a function with the same name as an intrinsic. #pragma redefine_extname __atomic_load_c SYMBOL_NAME(__atomic_load) #pragma redefine_extname __atomic_store_c SYMBOL_NAME(__atomic_store) #pragma redefine_extname __atomic_exchange_c SYMBOL_NAME(__atomic_exchange) #pragma redefine_extname __atomic_compare_exchange_c SYMBOL_NAME( \ __atomic_compare_exchange) /// Number of locks. This allocates one page on 32-bit platforms, two on /// 64-bit. This can be specified externally if a different trade between /// memory usage and contention probability is required for a given platform. #ifndef SPINLOCK_COUNT #define SPINLOCK_COUNT (1 << 10) #endif static const long SPINLOCK_MASK = SPINLOCK_COUNT - 1; //////////////////////////////////////////////////////////////////////////////// // Platform-specific lock implementation. Falls back to spinlocks if none is // defined. Each platform should define the Lock type, and corresponding // lock() and unlock() functions. //////////////////////////////////////////////////////////////////////////////// #ifdef __FreeBSD__ #include -#include #include +#include #include typedef struct _usem Lock; __inline static void unlock(Lock *l) { __c11_atomic_store((_Atomic(uint32_t) *)&l->_count, 1, __ATOMIC_RELEASE); __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); if (l->_has_waiters) _umtx_op(l, UMTX_OP_SEM_WAKE, 1, 0, 0); } __inline static void lock(Lock *l) { uint32_t old = 1; while (!__c11_atomic_compare_exchange_weak((_Atomic(uint32_t) *)&l->_count, &old, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { _umtx_op(l, UMTX_OP_SEM_WAIT, 0, 0, 0); old = 1; } } /// locks for atomic operations static Lock locks[SPINLOCK_COUNT] = {[0 ... SPINLOCK_COUNT - 1] = {0, 1, 0}}; #elif defined(__APPLE__) #include typedef OSSpinLock Lock; __inline static void unlock(Lock *l) { OSSpinLockUnlock(l); } /// Locks a lock. In the current implementation, this is potentially /// unbounded in the contended case. __inline static void lock(Lock *l) { OSSpinLockLock(l); } static Lock locks[SPINLOCK_COUNT]; // initialized to OS_SPINLOCK_INIT which is 0 #else typedef _Atomic(uintptr_t) Lock; /// Unlock a lock. This is a release operation. __inline static void unlock(Lock *l) { __c11_atomic_store(l, 0, __ATOMIC_RELEASE); } /// Locks a lock. In the current implementation, this is potentially /// unbounded in the contended case. __inline static void lock(Lock *l) { uintptr_t old = 0; while (!__c11_atomic_compare_exchange_weak(l, &old, 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) old = 0; } /// locks for atomic operations static Lock locks[SPINLOCK_COUNT]; #endif /// Returns a lock to use for a given pointer. static __inline Lock *lock_for_pointer(void *ptr) { intptr_t hash = (intptr_t)ptr; // Disregard the lowest 4 bits. We want all values that may be part of the // same memory operation to hash to the same value and therefore use the same // lock. hash >>= 4; // Use the next bits as the basis for the hash intptr_t low = hash & SPINLOCK_MASK; // Now use the high(er) set of bits to perturb the hash, so that we don't // get collisions from atomic fields in a single object hash >>= 16; hash ^= low; // Return a pointer to the word to use return locks + (hash & SPINLOCK_MASK); } -/// Macros for determining whether a size is lock free. Clang can not yet -/// codegen __atomic_is_lock_free(16), so for now we assume 16-byte values are -/// not lock free. +/// Macros for determining whether a size is lock free. #define IS_LOCK_FREE_1 __c11_atomic_is_lock_free(1) #define IS_LOCK_FREE_2 __c11_atomic_is_lock_free(2) #define IS_LOCK_FREE_4 __c11_atomic_is_lock_free(4) + +/// 32 bit PowerPC doesn't support 8-byte lock_free atomics +#if !defined(__powerpc64__) && defined(__powerpc__) +#define IS_LOCK_FREE_8 0 +#else #define IS_LOCK_FREE_8 __c11_atomic_is_lock_free(8) +#endif + +/// Clang can not yet codegen __atomic_is_lock_free(16), so for now we assume +/// 16-byte values are not lock free. #define IS_LOCK_FREE_16 0 /// Macro that calls the compiler-generated lock-free versions of functions /// when they exist. #define LOCK_FREE_CASES() \ do { \ switch (size) { \ case 1: \ if (IS_LOCK_FREE_1) { \ LOCK_FREE_ACTION(uint8_t); \ } \ break; \ case 2: \ if (IS_LOCK_FREE_2) { \ LOCK_FREE_ACTION(uint16_t); \ } \ break; \ case 4: \ if (IS_LOCK_FREE_4) { \ LOCK_FREE_ACTION(uint32_t); \ } \ break; \ case 8: \ if (IS_LOCK_FREE_8) { \ LOCK_FREE_ACTION(uint64_t); \ } \ break; \ case 16: \ if (IS_LOCK_FREE_16) { \ /* FIXME: __uint128_t isn't available on 32 bit platforms. \ LOCK_FREE_ACTION(__uint128_t);*/ \ } \ break; \ } \ } while (0) /// An atomic load operation. This is atomic with respect to the source /// pointer only. void __atomic_load_c(int size, void *src, void *dest, int model) { #define LOCK_FREE_ACTION(type) \ *((type *)dest) = __c11_atomic_load((_Atomic(type) *)src, model); \ return; LOCK_FREE_CASES(); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(src); lock(l); memcpy(dest, src, size); unlock(l); } /// An atomic store operation. This is atomic with respect to the destination /// pointer only. void __atomic_store_c(int size, void *dest, void *src, int model) { #define LOCK_FREE_ACTION(type) \ __c11_atomic_store((_Atomic(type) *)dest, *(type *)src, model); \ return; LOCK_FREE_CASES(); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(dest); lock(l); memcpy(dest, src, size); unlock(l); } /// Atomic compare and exchange operation. If the value at *ptr is identical /// to the value at *expected, then this copies value at *desired to *ptr. If /// they are not, then this stores the current value from *ptr in *expected. /// /// This function returns 1 if the exchange takes place or 0 if it fails. int __atomic_compare_exchange_c(int size, void *ptr, void *expected, void *desired, int success, int failure) { #define LOCK_FREE_ACTION(type) \ return __c11_atomic_compare_exchange_strong( \ (_Atomic(type) *)ptr, (type *)expected, *(type *)desired, success, \ failure) LOCK_FREE_CASES(); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(ptr); lock(l); if (memcmp(ptr, expected, size) == 0) { memcpy(ptr, desired, size); unlock(l); return 1; } memcpy(expected, ptr, size); unlock(l); return 0; } /// Performs an atomic exchange operation between two pointers. This is atomic /// with respect to the target address. void __atomic_exchange_c(int size, void *ptr, void *val, void *old, int model) { #define LOCK_FREE_ACTION(type) \ *(type *)old = \ __c11_atomic_exchange((_Atomic(type) *)ptr, *(type *)val, model); \ return; LOCK_FREE_CASES(); #undef LOCK_FREE_ACTION Lock *l = lock_for_pointer(ptr); lock(l); memcpy(old, ptr, size); memcpy(ptr, val, size); unlock(l); } //////////////////////////////////////////////////////////////////////////////// // Where the size is known at compile time, the compiler may emit calls to // specialised versions of the above functions. //////////////////////////////////////////////////////////////////////////////// #ifdef __SIZEOF_INT128__ #define OPTIMISED_CASES \ OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t) \ OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t) \ OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t) \ OPTIMISED_CASE(8, IS_LOCK_FREE_8, uint64_t) \ OPTIMISED_CASE(16, IS_LOCK_FREE_16, __uint128_t) #else #define OPTIMISED_CASES \ OPTIMISED_CASE(1, IS_LOCK_FREE_1, uint8_t) \ OPTIMISED_CASE(2, IS_LOCK_FREE_2, uint16_t) \ OPTIMISED_CASE(4, IS_LOCK_FREE_4, uint32_t) \ OPTIMISED_CASE(8, IS_LOCK_FREE_8, uint64_t) #endif #define OPTIMISED_CASE(n, lockfree, type) \ type __atomic_load_##n(type *src, int model) { \ if (lockfree) \ return __c11_atomic_load((_Atomic(type) *)src, model); \ Lock *l = lock_for_pointer(src); \ lock(l); \ type val = *src; \ unlock(l); \ return val; \ } OPTIMISED_CASES #undef OPTIMISED_CASE #define OPTIMISED_CASE(n, lockfree, type) \ void __atomic_store_##n(type *dest, type val, int model) { \ if (lockfree) { \ __c11_atomic_store((_Atomic(type) *)dest, val, model); \ return; \ } \ Lock *l = lock_for_pointer(dest); \ lock(l); \ *dest = val; \ unlock(l); \ return; \ } OPTIMISED_CASES #undef OPTIMISED_CASE #define OPTIMISED_CASE(n, lockfree, type) \ type __atomic_exchange_##n(type *dest, type val, int model) { \ if (lockfree) \ return __c11_atomic_exchange((_Atomic(type) *)dest, val, model); \ Lock *l = lock_for_pointer(dest); \ lock(l); \ type tmp = *dest; \ *dest = val; \ unlock(l); \ return tmp; \ } OPTIMISED_CASES #undef OPTIMISED_CASE #define OPTIMISED_CASE(n, lockfree, type) \ int __atomic_compare_exchange_##n(type *ptr, type *expected, type desired, \ int success, int failure) { \ if (lockfree) \ return __c11_atomic_compare_exchange_strong( \ (_Atomic(type) *)ptr, expected, desired, success, failure); \ Lock *l = lock_for_pointer(ptr); \ lock(l); \ if (*ptr == *expected) { \ *ptr = desired; \ unlock(l); \ return 1; \ } \ *expected = *ptr; \ unlock(l); \ return 0; \ } OPTIMISED_CASES #undef OPTIMISED_CASE //////////////////////////////////////////////////////////////////////////////// // Atomic read-modify-write operations for integers of various sizes. //////////////////////////////////////////////////////////////////////////////// #define ATOMIC_RMW(n, lockfree, type, opname, op) \ type __atomic_fetch_##opname##_##n(type *ptr, type val, int model) { \ if (lockfree) \ return __c11_atomic_fetch_##opname((_Atomic(type) *)ptr, val, model); \ Lock *l = lock_for_pointer(ptr); \ lock(l); \ type tmp = *ptr; \ *ptr = tmp op val; \ unlock(l); \ return tmp; \ } #define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, add, +) OPTIMISED_CASES #undef OPTIMISED_CASE #define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, sub, -) OPTIMISED_CASES #undef OPTIMISED_CASE #define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, and, &) OPTIMISED_CASES #undef OPTIMISED_CASE #define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, or, |) OPTIMISED_CASES #undef OPTIMISED_CASE #define OPTIMISED_CASE(n, lockfree, type) ATOMIC_RMW(n, lockfree, type, xor, ^) OPTIMISED_CASES #undef OPTIMISED_CASE Index: stable/12/contrib/llvm-project/compiler-rt =================================================================== --- stable/12/contrib/llvm-project/compiler-rt (revision 356466) +++ stable/12/contrib/llvm-project/compiler-rt (revision 356467) Property changes on: stable/12/contrib/llvm-project/compiler-rt ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/contrib/llvm-project/compiler-rt:r356104,356112 Index: stable/12/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp =================================================================== --- stable/12/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp (revision 356466) +++ stable/12/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCISelLowering.cpp (revision 356467) @@ -1,15312 +1,15317 @@ //===-- PPCISelLowering.cpp - PPC DAG Lowering Implementation -------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file implements the PPCISelLowering class. // //===----------------------------------------------------------------------===// #include "PPCISelLowering.h" #include "MCTargetDesc/PPCPredicates.h" #include "PPC.h" #include "PPCCCState.h" #include "PPCCallingConv.h" #include "PPCFrameLowering.h" #include "PPCInstrInfo.h" #include "PPCMachineFunctionInfo.h" #include "PPCPerfectShuffle.h" #include "PPCRegisterInfo.h" #include "PPCSubtarget.h" #include "PPCTargetMachine.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/None.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/CodeGen/CallingConvLower.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/RuntimeLibcalls.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/ValueTypes.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/Use.h" #include "llvm/IR/Value.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSymbolXCOFF.h" #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/BranchProbability.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Format.h" #include "llvm/Support/KnownBits.h" #include "llvm/Support/MachineValueType.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include #include #include #include #include #include #include using namespace llvm; #define DEBUG_TYPE "ppc-lowering" static cl::opt DisablePPCPreinc("disable-ppc-preinc", cl::desc("disable preincrement load/store generation on PPC"), cl::Hidden); static cl::opt DisableILPPref("disable-ppc-ilp-pref", cl::desc("disable setting the node scheduling preference to ILP on PPC"), cl::Hidden); static cl::opt DisablePPCUnaligned("disable-ppc-unaligned", cl::desc("disable unaligned load/store generation on PPC"), cl::Hidden); static cl::opt DisableSCO("disable-ppc-sco", cl::desc("disable sibling call optimization on ppc"), cl::Hidden); static cl::opt DisableInnermostLoopAlign32("disable-ppc-innermost-loop-align32", cl::desc("don't always align innermost loop to 32 bytes on ppc"), cl::Hidden); static cl::opt EnableQuadPrecision("enable-ppc-quad-precision", cl::desc("enable quad precision float support on ppc"), cl::Hidden); STATISTIC(NumTailCalls, "Number of tail calls"); STATISTIC(NumSiblingCalls, "Number of sibling calls"); static bool isNByteElemShuffleMask(ShuffleVectorSDNode *, unsigned, int); static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl); // FIXME: Remove this once the bug has been fixed! extern cl::opt ANDIGlueBug; PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, const PPCSubtarget &STI) : TargetLowering(TM), Subtarget(STI) { // Use _setjmp/_longjmp instead of setjmp/longjmp. setUseUnderscoreSetJmp(true); setUseUnderscoreLongJmp(true); // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all // arguments are at least 4/8 bytes aligned. bool isPPC64 = Subtarget.isPPC64(); setMinStackArgumentAlignment(isPPC64 ? 8:4); // Set up the register classes. addRegisterClass(MVT::i32, &PPC::GPRCRegClass); if (!useSoftFloat()) { if (hasSPE()) { addRegisterClass(MVT::f32, &PPC::SPE4RCRegClass); addRegisterClass(MVT::f64, &PPC::SPERCRegClass); } else { addRegisterClass(MVT::f32, &PPC::F4RCRegClass); addRegisterClass(MVT::f64, &PPC::F8RCRegClass); } } // Match BITREVERSE to customized fast code sequence in the td file. setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); // Sub-word ATOMIC_CMP_SWAP need to ensure that the input is zero-extended. setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); // PowerPC has an i16 but no i8 (or i1) SEXTLOAD. for (MVT VT : MVT::integer_valuetypes()) { setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Expand); } setTruncStoreAction(MVT::f64, MVT::f32, Expand); // PowerPC has pre-inc load and store's. setIndexedLoadAction(ISD::PRE_INC, MVT::i1, Legal); setIndexedLoadAction(ISD::PRE_INC, MVT::i8, Legal); setIndexedLoadAction(ISD::PRE_INC, MVT::i16, Legal); setIndexedLoadAction(ISD::PRE_INC, MVT::i32, Legal); setIndexedLoadAction(ISD::PRE_INC, MVT::i64, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::i1, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::i8, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::i16, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::i32, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::i64, Legal); if (!Subtarget.hasSPE()) { setIndexedLoadAction(ISD::PRE_INC, MVT::f32, Legal); setIndexedLoadAction(ISD::PRE_INC, MVT::f64, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::f32, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::f64, Legal); } // PowerPC uses ADDC/ADDE/SUBC/SUBE to propagate carry. const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; for (MVT VT : ScalarIntVTs) { setOperationAction(ISD::ADDC, VT, Legal); setOperationAction(ISD::ADDE, VT, Legal); setOperationAction(ISD::SUBC, VT, Legal); setOperationAction(ISD::SUBE, VT, Legal); } if (Subtarget.useCRBits()) { setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); if (isPPC64 || Subtarget.hasFPCVT()) { setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote); AddPromotedToType (ISD::SINT_TO_FP, MVT::i1, isPPC64 ? MVT::i64 : MVT::i32); setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote); AddPromotedToType(ISD::UINT_TO_FP, MVT::i1, isPPC64 ? MVT::i64 : MVT::i32); } else { setOperationAction(ISD::SINT_TO_FP, MVT::i1, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::i1, Custom); } // PowerPC does not support direct load/store of condition registers. setOperationAction(ISD::LOAD, MVT::i1, Custom); setOperationAction(ISD::STORE, MVT::i1, Custom); // FIXME: Remove this once the ANDI glue bug is fixed: if (ANDIGlueBug) setOperationAction(ISD::TRUNCATE, MVT::i1, Custom); for (MVT VT : MVT::integer_valuetypes()) { setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); setTruncStoreAction(VT, MVT::i1, Expand); } addRegisterClass(MVT::i1, &PPC::CRBITRCRegClass); } // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on // PPC (the libcall is not available). setOperationAction(ISD::FP_TO_SINT, MVT::ppcf128, Custom); setOperationAction(ISD::FP_TO_UINT, MVT::ppcf128, Custom); // We do not currently implement these libm ops for PowerPC. setOperationAction(ISD::FFLOOR, MVT::ppcf128, Expand); setOperationAction(ISD::FCEIL, MVT::ppcf128, Expand); setOperationAction(ISD::FTRUNC, MVT::ppcf128, Expand); setOperationAction(ISD::FRINT, MVT::ppcf128, Expand); setOperationAction(ISD::FNEARBYINT, MVT::ppcf128, Expand); setOperationAction(ISD::FREM, MVT::ppcf128, Expand); // PowerPC has no SREM/UREM instructions unless we are on P9 // On P9 we may use a hardware instruction to compute the remainder. // The instructions are not legalized directly because in the cases where the // result of both the remainder and the division is required it is more // efficient to compute the remainder from the result of the division rather // than use the remainder instruction. if (Subtarget.isISA3_0()) { setOperationAction(ISD::SREM, MVT::i32, Custom); setOperationAction(ISD::UREM, MVT::i32, Custom); setOperationAction(ISD::SREM, MVT::i64, Custom); setOperationAction(ISD::UREM, MVT::i64, Custom); } else { setOperationAction(ISD::SREM, MVT::i32, Expand); setOperationAction(ISD::UREM, MVT::i32, Expand); setOperationAction(ISD::SREM, MVT::i64, Expand); setOperationAction(ISD::UREM, MVT::i64, Expand); } // Don't use SMUL_LOHI/UMUL_LOHI or SDIVREM/UDIVREM to lower SREM/UREM. setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand); setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand); setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand); setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); setOperationAction(ISD::UDIVREM, MVT::i32, Expand); setOperationAction(ISD::SDIVREM, MVT::i32, Expand); setOperationAction(ISD::UDIVREM, MVT::i64, Expand); setOperationAction(ISD::SDIVREM, MVT::i64, Expand); // We don't support sin/cos/sqrt/fmod/pow setOperationAction(ISD::FSIN , MVT::f64, Expand); setOperationAction(ISD::FCOS , MVT::f64, Expand); setOperationAction(ISD::FSINCOS, MVT::f64, Expand); setOperationAction(ISD::FREM , MVT::f64, Expand); setOperationAction(ISD::FPOW , MVT::f64, Expand); setOperationAction(ISD::FSIN , MVT::f32, Expand); setOperationAction(ISD::FCOS , MVT::f32, Expand); setOperationAction(ISD::FSINCOS, MVT::f32, Expand); setOperationAction(ISD::FREM , MVT::f32, Expand); setOperationAction(ISD::FPOW , MVT::f32, Expand); if (Subtarget.hasSPE()) { setOperationAction(ISD::FMA , MVT::f64, Expand); setOperationAction(ISD::FMA , MVT::f32, Expand); } else { setOperationAction(ISD::FMA , MVT::f64, Legal); setOperationAction(ISD::FMA , MVT::f32, Legal); } setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom); // If we're enabling GP optimizations, use hardware square root if (!Subtarget.hasFSQRT() && !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTE() && Subtarget.hasFRE())) setOperationAction(ISD::FSQRT, MVT::f64, Expand); if (!Subtarget.hasFSQRT() && !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTES() && Subtarget.hasFRES())) setOperationAction(ISD::FSQRT, MVT::f32, Expand); if (Subtarget.hasFCPSGN()) { setOperationAction(ISD::FCOPYSIGN, MVT::f64, Legal); setOperationAction(ISD::FCOPYSIGN, MVT::f32, Legal); } else { setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); } if (Subtarget.hasFPRND()) { setOperationAction(ISD::FFLOOR, MVT::f64, Legal); setOperationAction(ISD::FCEIL, MVT::f64, Legal); setOperationAction(ISD::FTRUNC, MVT::f64, Legal); setOperationAction(ISD::FROUND, MVT::f64, Legal); setOperationAction(ISD::FFLOOR, MVT::f32, Legal); setOperationAction(ISD::FCEIL, MVT::f32, Legal); setOperationAction(ISD::FTRUNC, MVT::f32, Legal); setOperationAction(ISD::FROUND, MVT::f32, Legal); } // PowerPC does not have BSWAP, but we can use vector BSWAP instruction xxbrd // to speed up scalar BSWAP64. // CTPOP or CTTZ were introduced in P8/P9 respectively setOperationAction(ISD::BSWAP, MVT::i32 , Expand); if (Subtarget.hasP9Vector()) setOperationAction(ISD::BSWAP, MVT::i64 , Custom); else setOperationAction(ISD::BSWAP, MVT::i64 , Expand); if (Subtarget.isISA3_0()) { setOperationAction(ISD::CTTZ , MVT::i32 , Legal); setOperationAction(ISD::CTTZ , MVT::i64 , Legal); } else { setOperationAction(ISD::CTTZ , MVT::i32 , Expand); setOperationAction(ISD::CTTZ , MVT::i64 , Expand); } if (Subtarget.hasPOPCNTD() == PPCSubtarget::POPCNTD_Fast) { setOperationAction(ISD::CTPOP, MVT::i32 , Legal); setOperationAction(ISD::CTPOP, MVT::i64 , Legal); } else { setOperationAction(ISD::CTPOP, MVT::i32 , Expand); setOperationAction(ISD::CTPOP, MVT::i64 , Expand); } // PowerPC does not have ROTR setOperationAction(ISD::ROTR, MVT::i32 , Expand); setOperationAction(ISD::ROTR, MVT::i64 , Expand); if (!Subtarget.useCRBits()) { // PowerPC does not have Select setOperationAction(ISD::SELECT, MVT::i32, Expand); setOperationAction(ISD::SELECT, MVT::i64, Expand); setOperationAction(ISD::SELECT, MVT::f32, Expand); setOperationAction(ISD::SELECT, MVT::f64, Expand); } // PowerPC wants to turn select_cc of FP into fsel when possible. setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); // PowerPC wants to optimize integer setcc a bit if (!Subtarget.useCRBits()) setOperationAction(ISD::SETCC, MVT::i32, Custom); // PowerPC does not have BRCOND which requires SetCC if (!Subtarget.useCRBits()) setOperationAction(ISD::BRCOND, MVT::Other, Expand); setOperationAction(ISD::BR_JT, MVT::Other, Expand); if (Subtarget.hasSPE()) { // SPE has built-in conversions setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal); setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal); setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal); } else { // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores. setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); // PowerPC does not have [U|S]INT_TO_FP setOperationAction(ISD::SINT_TO_FP, MVT::i32, Expand); setOperationAction(ISD::UINT_TO_FP, MVT::i32, Expand); } if (Subtarget.hasDirectMove() && isPPC64) { setOperationAction(ISD::BITCAST, MVT::f32, Legal); setOperationAction(ISD::BITCAST, MVT::i32, Legal); setOperationAction(ISD::BITCAST, MVT::i64, Legal); setOperationAction(ISD::BITCAST, MVT::f64, Legal); } else { setOperationAction(ISD::BITCAST, MVT::f32, Expand); setOperationAction(ISD::BITCAST, MVT::i32, Expand); setOperationAction(ISD::BITCAST, MVT::i64, Expand); setOperationAction(ISD::BITCAST, MVT::f64, Expand); } // We cannot sextinreg(i1). Expand to shifts. setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support // SjLj exception handling but a light-weight setjmp/longjmp replacement to // support continuation, user-level threading, and etc.. As a result, no // other SjLj exception interfaces are implemented and please don't build // your own exception handling based on them. // LLVM/Clang supports zero-cost DWARF exception handling. setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); // We want to legalize GlobalAddress and ConstantPool nodes into the // appropriate instructions to materialize the address. setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom); setOperationAction(ISD::BlockAddress, MVT::i32, Custom); setOperationAction(ISD::ConstantPool, MVT::i32, Custom); setOperationAction(ISD::JumpTable, MVT::i32, Custom); setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); setOperationAction(ISD::BlockAddress, MVT::i64, Custom); setOperationAction(ISD::ConstantPool, MVT::i64, Custom); setOperationAction(ISD::JumpTable, MVT::i64, Custom); // TRAP is legal. setOperationAction(ISD::TRAP, MVT::Other, Legal); // TRAMPOLINE is custom lowered. setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom); setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom); // VASTART needs to be custom lowered to use the VarArgsFrameIndex setOperationAction(ISD::VASTART , MVT::Other, Custom); if (Subtarget.isSVR4ABI()) { if (isPPC64) { // VAARG always uses double-word chunks, so promote anything smaller. setOperationAction(ISD::VAARG, MVT::i1, Promote); AddPromotedToType (ISD::VAARG, MVT::i1, MVT::i64); setOperationAction(ISD::VAARG, MVT::i8, Promote); AddPromotedToType (ISD::VAARG, MVT::i8, MVT::i64); setOperationAction(ISD::VAARG, MVT::i16, Promote); AddPromotedToType (ISD::VAARG, MVT::i16, MVT::i64); setOperationAction(ISD::VAARG, MVT::i32, Promote); AddPromotedToType (ISD::VAARG, MVT::i32, MVT::i64); setOperationAction(ISD::VAARG, MVT::Other, Expand); } else { // VAARG is custom lowered with the 32-bit SVR4 ABI. setOperationAction(ISD::VAARG, MVT::Other, Custom); setOperationAction(ISD::VAARG, MVT::i64, Custom); } } else setOperationAction(ISD::VAARG, MVT::Other, Expand); if (Subtarget.isSVR4ABI() && !isPPC64) // VACOPY is custom lowered with the 32-bit SVR4 ABI. setOperationAction(ISD::VACOPY , MVT::Other, Custom); else setOperationAction(ISD::VACOPY , MVT::Other, Expand); // Use the default implementation. setOperationAction(ISD::VAEND , MVT::Other, Expand); setOperationAction(ISD::STACKSAVE , MVT::Other, Expand); setOperationAction(ISD::STACKRESTORE , MVT::Other, Custom); setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Custom); setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64 , Custom); setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i32, Custom); setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i64, Custom); setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom); setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom); // We want to custom lower some of our intrinsics. setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); // To handle counter-based loop conditions. setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i1, Custom); setOperationAction(ISD::INTRINSIC_VOID, MVT::i8, Custom); setOperationAction(ISD::INTRINSIC_VOID, MVT::i16, Custom); setOperationAction(ISD::INTRINSIC_VOID, MVT::i32, Custom); setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); // Comparisons that require checking two conditions. if (Subtarget.hasSPE()) { setCondCodeAction(ISD::SETO, MVT::f32, Expand); setCondCodeAction(ISD::SETO, MVT::f64, Expand); setCondCodeAction(ISD::SETUO, MVT::f32, Expand); setCondCodeAction(ISD::SETUO, MVT::f64, Expand); } setCondCodeAction(ISD::SETULT, MVT::f32, Expand); setCondCodeAction(ISD::SETULT, MVT::f64, Expand); setCondCodeAction(ISD::SETUGT, MVT::f32, Expand); setCondCodeAction(ISD::SETUGT, MVT::f64, Expand); setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand); setCondCodeAction(ISD::SETUEQ, MVT::f64, Expand); setCondCodeAction(ISD::SETOGE, MVT::f32, Expand); setCondCodeAction(ISD::SETOGE, MVT::f64, Expand); setCondCodeAction(ISD::SETOLE, MVT::f32, Expand); setCondCodeAction(ISD::SETOLE, MVT::f64, Expand); setCondCodeAction(ISD::SETONE, MVT::f32, Expand); setCondCodeAction(ISD::SETONE, MVT::f64, Expand); if (Subtarget.has64BitSupport()) { // They also have instructions for converting between i64 and fp. setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); setOperationAction(ISD::FP_TO_UINT, MVT::i64, Expand); setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); // This is just the low 32 bits of a (signed) fp->i64 conversion. // We cannot do this with Promote because i64 is not a legal type. setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); if (Subtarget.hasLFIWAX() || Subtarget.isPPC64()) setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); } else { // PowerPC does not have FP_TO_UINT on 32-bit implementations. if (Subtarget.hasSPE()) setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal); else setOperationAction(ISD::FP_TO_UINT, MVT::i32, Expand); } // With the instructions enabled under FPCVT, we can do everything. if (Subtarget.hasFPCVT()) { if (Subtarget.has64BitSupport()) { setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); } setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); } if (Subtarget.use64BitRegs()) { // 64-bit PowerPC implementations can support i64 types directly addRegisterClass(MVT::i64, &PPC::G8RCRegClass); // BUILD_PAIR can't be handled natively, and should be expanded to shl/or setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); // 64-bit PowerPC wants to expand i128 shifts itself. setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); } else { // 32-bit PowerPC wants to expand i64 shifts itself. setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom); setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom); setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); } if (Subtarget.hasAltivec()) { // First set operation action for all vector types to expand. Then we // will selectively turn on ones that can be effectively codegen'd. for (MVT VT : MVT::vector_valuetypes()) { // add/sub are legal for all supported vector VT's. setOperationAction(ISD::ADD, VT, Legal); setOperationAction(ISD::SUB, VT, Legal); // For v2i64, these are only valid with P8Vector. This is corrected after // the loop. setOperationAction(ISD::SMAX, VT, Legal); setOperationAction(ISD::SMIN, VT, Legal); setOperationAction(ISD::UMAX, VT, Legal); setOperationAction(ISD::UMIN, VT, Legal); if (Subtarget.hasVSX()) { setOperationAction(ISD::FMAXNUM, VT, Legal); setOperationAction(ISD::FMINNUM, VT, Legal); } // Vector instructions introduced in P8 if (Subtarget.hasP8Altivec() && (VT.SimpleTy != MVT::v1i128)) { setOperationAction(ISD::CTPOP, VT, Legal); setOperationAction(ISD::CTLZ, VT, Legal); } else { setOperationAction(ISD::CTPOP, VT, Expand); setOperationAction(ISD::CTLZ, VT, Expand); } // Vector instructions introduced in P9 if (Subtarget.hasP9Altivec() && (VT.SimpleTy != MVT::v1i128)) setOperationAction(ISD::CTTZ, VT, Legal); else setOperationAction(ISD::CTTZ, VT, Expand); // We promote all shuffles to v16i8. setOperationAction(ISD::VECTOR_SHUFFLE, VT, Promote); AddPromotedToType (ISD::VECTOR_SHUFFLE, VT, MVT::v16i8); // We promote all non-typed operations to v4i32. setOperationAction(ISD::AND , VT, Promote); AddPromotedToType (ISD::AND , VT, MVT::v4i32); setOperationAction(ISD::OR , VT, Promote); AddPromotedToType (ISD::OR , VT, MVT::v4i32); setOperationAction(ISD::XOR , VT, Promote); AddPromotedToType (ISD::XOR , VT, MVT::v4i32); setOperationAction(ISD::LOAD , VT, Promote); AddPromotedToType (ISD::LOAD , VT, MVT::v4i32); setOperationAction(ISD::SELECT, VT, Promote); AddPromotedToType (ISD::SELECT, VT, MVT::v4i32); setOperationAction(ISD::VSELECT, VT, Legal); setOperationAction(ISD::SELECT_CC, VT, Promote); AddPromotedToType (ISD::SELECT_CC, VT, MVT::v4i32); setOperationAction(ISD::STORE, VT, Promote); AddPromotedToType (ISD::STORE, VT, MVT::v4i32); // No other operations are legal. setOperationAction(ISD::MUL , VT, Expand); setOperationAction(ISD::SDIV, VT, Expand); setOperationAction(ISD::SREM, VT, Expand); setOperationAction(ISD::UDIV, VT, Expand); setOperationAction(ISD::UREM, VT, Expand); setOperationAction(ISD::FDIV, VT, Expand); setOperationAction(ISD::FREM, VT, Expand); setOperationAction(ISD::FNEG, VT, Expand); setOperationAction(ISD::FSQRT, VT, Expand); setOperationAction(ISD::FLOG, VT, Expand); setOperationAction(ISD::FLOG10, VT, Expand); setOperationAction(ISD::FLOG2, VT, Expand); setOperationAction(ISD::FEXP, VT, Expand); setOperationAction(ISD::FEXP2, VT, Expand); setOperationAction(ISD::FSIN, VT, Expand); setOperationAction(ISD::FCOS, VT, Expand); setOperationAction(ISD::FABS, VT, Expand); setOperationAction(ISD::FFLOOR, VT, Expand); setOperationAction(ISD::FCEIL, VT, Expand); setOperationAction(ISD::FTRUNC, VT, Expand); setOperationAction(ISD::FRINT, VT, Expand); setOperationAction(ISD::FNEARBYINT, VT, Expand); setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Expand); setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand); setOperationAction(ISD::BUILD_VECTOR, VT, Expand); setOperationAction(ISD::MULHU, VT, Expand); setOperationAction(ISD::MULHS, VT, Expand); setOperationAction(ISD::UMUL_LOHI, VT, Expand); setOperationAction(ISD::SMUL_LOHI, VT, Expand); setOperationAction(ISD::UDIVREM, VT, Expand); setOperationAction(ISD::SDIVREM, VT, Expand); setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand); setOperationAction(ISD::FPOW, VT, Expand); setOperationAction(ISD::BSWAP, VT, Expand); setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); setOperationAction(ISD::ROTL, VT, Expand); setOperationAction(ISD::ROTR, VT, Expand); for (MVT InnerVT : MVT::vector_valuetypes()) { setTruncStoreAction(VT, InnerVT, Expand); setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); } } if (!Subtarget.hasP8Vector()) { setOperationAction(ISD::SMAX, MVT::v2i64, Expand); setOperationAction(ISD::SMIN, MVT::v2i64, Expand); setOperationAction(ISD::UMAX, MVT::v2i64, Expand); setOperationAction(ISD::UMIN, MVT::v2i64, Expand); } for (auto VT : {MVT::v2i64, MVT::v4i32, MVT::v8i16, MVT::v16i8}) setOperationAction(ISD::ABS, VT, Custom); // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle // with merges, splats, etc. setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i8, Custom); // Vector truncates to sub-word integer that fit in an Altivec/VSX register // are cheap, so handle them before they get expanded to scalar. setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom); setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom); setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom); setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom); setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom); setOperationAction(ISD::AND , MVT::v4i32, Legal); setOperationAction(ISD::OR , MVT::v4i32, Legal); setOperationAction(ISD::XOR , MVT::v4i32, Legal); setOperationAction(ISD::LOAD , MVT::v4i32, Legal); setOperationAction(ISD::SELECT, MVT::v4i32, Subtarget.useCRBits() ? Legal : Expand); setOperationAction(ISD::STORE , MVT::v4i32, Legal); setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal); setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal); setOperationAction(ISD::FCEIL, MVT::v4f32, Legal); setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal); // Without hasP8Altivec set, v2i64 SMAX isn't available. // But ABS custom lowering requires SMAX support. if (!Subtarget.hasP8Altivec()) setOperationAction(ISD::ABS, MVT::v2i64, Expand); addRegisterClass(MVT::v4f32, &PPC::VRRCRegClass); addRegisterClass(MVT::v4i32, &PPC::VRRCRegClass); addRegisterClass(MVT::v8i16, &PPC::VRRCRegClass); addRegisterClass(MVT::v16i8, &PPC::VRRCRegClass); setOperationAction(ISD::MUL, MVT::v4f32, Legal); setOperationAction(ISD::FMA, MVT::v4f32, Legal); if (TM.Options.UnsafeFPMath || Subtarget.hasVSX()) { setOperationAction(ISD::FDIV, MVT::v4f32, Legal); setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); } if (Subtarget.hasP8Altivec()) setOperationAction(ISD::MUL, MVT::v4i32, Legal); else setOperationAction(ISD::MUL, MVT::v4i32, Custom); setOperationAction(ISD::MUL, MVT::v8i16, Custom); setOperationAction(ISD::MUL, MVT::v16i8, Custom); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Custom); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Custom); setOperationAction(ISD::BUILD_VECTOR, MVT::v16i8, Custom); setOperationAction(ISD::BUILD_VECTOR, MVT::v8i16, Custom); setOperationAction(ISD::BUILD_VECTOR, MVT::v4i32, Custom); setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); // Altivec does not contain unordered floating-point compare instructions setCondCodeAction(ISD::SETUO, MVT::v4f32, Expand); setCondCodeAction(ISD::SETUEQ, MVT::v4f32, Expand); setCondCodeAction(ISD::SETO, MVT::v4f32, Expand); setCondCodeAction(ISD::SETONE, MVT::v4f32, Expand); if (Subtarget.hasVSX()) { setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal); if (Subtarget.hasP8Vector()) { setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Legal); } if (Subtarget.hasDirectMove() && isPPC64) { setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Legal); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Legal); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Legal); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal); } setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal); setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal); setOperationAction(ISD::FCEIL, MVT::v2f64, Legal); setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal); setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal); setOperationAction(ISD::FROUND, MVT::v2f64, Legal); setOperationAction(ISD::FROUND, MVT::v4f32, Legal); setOperationAction(ISD::MUL, MVT::v2f64, Legal); setOperationAction(ISD::FMA, MVT::v2f64, Legal); setOperationAction(ISD::FDIV, MVT::v2f64, Legal); setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); // Share the Altivec comparison restrictions. setCondCodeAction(ISD::SETUO, MVT::v2f64, Expand); setCondCodeAction(ISD::SETUEQ, MVT::v2f64, Expand); setCondCodeAction(ISD::SETO, MVT::v2f64, Expand); setCondCodeAction(ISD::SETONE, MVT::v2f64, Expand); setOperationAction(ISD::LOAD, MVT::v2f64, Legal); setOperationAction(ISD::STORE, MVT::v2f64, Legal); setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Legal); if (Subtarget.hasP8Vector()) addRegisterClass(MVT::f32, &PPC::VSSRCRegClass); addRegisterClass(MVT::f64, &PPC::VSFRCRegClass); addRegisterClass(MVT::v4i32, &PPC::VSRCRegClass); addRegisterClass(MVT::v4f32, &PPC::VSRCRegClass); addRegisterClass(MVT::v2f64, &PPC::VSRCRegClass); if (Subtarget.hasP8Altivec()) { setOperationAction(ISD::SHL, MVT::v2i64, Legal); setOperationAction(ISD::SRA, MVT::v2i64, Legal); setOperationAction(ISD::SRL, MVT::v2i64, Legal); // 128 bit shifts can be accomplished via 3 instructions for SHL and // SRL, but not for SRA because of the instructions available: // VS{RL} and VS{RL}O. However due to direct move costs, it's not worth // doing setOperationAction(ISD::SHL, MVT::v1i128, Expand); setOperationAction(ISD::SRL, MVT::v1i128, Expand); setOperationAction(ISD::SRA, MVT::v1i128, Expand); setOperationAction(ISD::SETCC, MVT::v2i64, Legal); } else { setOperationAction(ISD::SHL, MVT::v2i64, Expand); setOperationAction(ISD::SRA, MVT::v2i64, Expand); setOperationAction(ISD::SRL, MVT::v2i64, Expand); setOperationAction(ISD::SETCC, MVT::v2i64, Custom); // VSX v2i64 only supports non-arithmetic operations. setOperationAction(ISD::ADD, MVT::v2i64, Expand); setOperationAction(ISD::SUB, MVT::v2i64, Expand); } setOperationAction(ISD::LOAD, MVT::v2i64, Promote); AddPromotedToType (ISD::LOAD, MVT::v2i64, MVT::v2f64); setOperationAction(ISD::STORE, MVT::v2i64, Promote); AddPromotedToType (ISD::STORE, MVT::v2i64, MVT::v2f64); setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Legal); setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal); setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal); setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal); setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal); // Custom handling for partial vectors of integers converted to // floating point. We already have optimal handling for v2i32 through // the DAG combine, so those aren't necessary. setOperationAction(ISD::UINT_TO_FP, MVT::v2i8, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::v2i16, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::v2i8, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::v4i8, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::v2i16, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom); setOperationAction(ISD::FNEG, MVT::v4f32, Legal); setOperationAction(ISD::FNEG, MVT::v2f64, Legal); setOperationAction(ISD::FABS, MVT::v4f32, Legal); setOperationAction(ISD::FABS, MVT::v2f64, Legal); setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal); setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Legal); if (Subtarget.hasDirectMove()) setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); addRegisterClass(MVT::v2i64, &PPC::VSRCRegClass); } if (Subtarget.hasP8Altivec()) { addRegisterClass(MVT::v2i64, &PPC::VRRCRegClass); addRegisterClass(MVT::v1i128, &PPC::VRRCRegClass); } if (Subtarget.hasP9Vector()) { setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); // 128 bit shifts can be accomplished via 3 instructions for SHL and // SRL, but not for SRA because of the instructions available: // VS{RL} and VS{RL}O. setOperationAction(ISD::SHL, MVT::v1i128, Legal); setOperationAction(ISD::SRL, MVT::v1i128, Legal); setOperationAction(ISD::SRA, MVT::v1i128, Expand); if (EnableQuadPrecision) { addRegisterClass(MVT::f128, &PPC::VRRCRegClass); setOperationAction(ISD::FADD, MVT::f128, Legal); setOperationAction(ISD::FSUB, MVT::f128, Legal); setOperationAction(ISD::FDIV, MVT::f128, Legal); setOperationAction(ISD::FMUL, MVT::f128, Legal); setOperationAction(ISD::FP_EXTEND, MVT::f128, Legal); // No extending loads to f128 on PPC. for (MVT FPT : MVT::fp_valuetypes()) setLoadExtAction(ISD::EXTLOAD, MVT::f128, FPT, Expand); setOperationAction(ISD::FMA, MVT::f128, Legal); setCondCodeAction(ISD::SETULT, MVT::f128, Expand); setCondCodeAction(ISD::SETUGT, MVT::f128, Expand); setCondCodeAction(ISD::SETUEQ, MVT::f128, Expand); setCondCodeAction(ISD::SETOGE, MVT::f128, Expand); setCondCodeAction(ISD::SETOLE, MVT::f128, Expand); setCondCodeAction(ISD::SETONE, MVT::f128, Expand); setOperationAction(ISD::FTRUNC, MVT::f128, Legal); setOperationAction(ISD::FRINT, MVT::f128, Legal); setOperationAction(ISD::FFLOOR, MVT::f128, Legal); setOperationAction(ISD::FCEIL, MVT::f128, Legal); setOperationAction(ISD::FNEARBYINT, MVT::f128, Legal); setOperationAction(ISD::FROUND, MVT::f128, Legal); setOperationAction(ISD::SELECT, MVT::f128, Expand); setOperationAction(ISD::FP_ROUND, MVT::f64, Legal); setOperationAction(ISD::FP_ROUND, MVT::f32, Legal); setTruncStoreAction(MVT::f128, MVT::f64, Expand); setTruncStoreAction(MVT::f128, MVT::f32, Expand); setOperationAction(ISD::BITCAST, MVT::i128, Custom); // No implementation for these ops for PowerPC. setOperationAction(ISD::FSIN , MVT::f128, Expand); setOperationAction(ISD::FCOS , MVT::f128, Expand); setOperationAction(ISD::FPOW, MVT::f128, Expand); setOperationAction(ISD::FPOWI, MVT::f128, Expand); setOperationAction(ISD::FREM, MVT::f128, Expand); } setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom); } if (Subtarget.hasP9Altivec()) { setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); } } if (Subtarget.hasQPX()) { setOperationAction(ISD::FADD, MVT::v4f64, Legal); setOperationAction(ISD::FSUB, MVT::v4f64, Legal); setOperationAction(ISD::FMUL, MVT::v4f64, Legal); setOperationAction(ISD::FREM, MVT::v4f64, Expand); setOperationAction(ISD::FCOPYSIGN, MVT::v4f64, Legal); setOperationAction(ISD::FGETSIGN, MVT::v4f64, Expand); setOperationAction(ISD::LOAD , MVT::v4f64, Custom); setOperationAction(ISD::STORE , MVT::v4f64, Custom); setTruncStoreAction(MVT::v4f64, MVT::v4f32, Custom); setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Custom); if (!Subtarget.useCRBits()) setOperationAction(ISD::SELECT, MVT::v4f64, Expand); setOperationAction(ISD::VSELECT, MVT::v4f64, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f64, Legal); setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f64, Expand); setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f64, Expand); setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f64, Expand); setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f64, Custom); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f64, Legal); setOperationAction(ISD::BUILD_VECTOR, MVT::v4f64, Custom); setOperationAction(ISD::FP_TO_SINT , MVT::v4f64, Legal); setOperationAction(ISD::FP_TO_UINT , MVT::v4f64, Expand); setOperationAction(ISD::FP_ROUND , MVT::v4f32, Legal); setOperationAction(ISD::FP_ROUND_INREG , MVT::v4f32, Expand); setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Legal); setOperationAction(ISD::FNEG , MVT::v4f64, Legal); setOperationAction(ISD::FABS , MVT::v4f64, Legal); setOperationAction(ISD::FSIN , MVT::v4f64, Expand); setOperationAction(ISD::FCOS , MVT::v4f64, Expand); setOperationAction(ISD::FPOW , MVT::v4f64, Expand); setOperationAction(ISD::FLOG , MVT::v4f64, Expand); setOperationAction(ISD::FLOG2 , MVT::v4f64, Expand); setOperationAction(ISD::FLOG10 , MVT::v4f64, Expand); setOperationAction(ISD::FEXP , MVT::v4f64, Expand); setOperationAction(ISD::FEXP2 , MVT::v4f64, Expand); setOperationAction(ISD::FMINNUM, MVT::v4f64, Legal); setOperationAction(ISD::FMAXNUM, MVT::v4f64, Legal); setIndexedLoadAction(ISD::PRE_INC, MVT::v4f64, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::v4f64, Legal); addRegisterClass(MVT::v4f64, &PPC::QFRCRegClass); setOperationAction(ISD::FADD, MVT::v4f32, Legal); setOperationAction(ISD::FSUB, MVT::v4f32, Legal); setOperationAction(ISD::FMUL, MVT::v4f32, Legal); setOperationAction(ISD::FREM, MVT::v4f32, Expand); setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal); setOperationAction(ISD::FGETSIGN, MVT::v4f32, Expand); setOperationAction(ISD::LOAD , MVT::v4f32, Custom); setOperationAction(ISD::STORE , MVT::v4f32, Custom); if (!Subtarget.useCRBits()) setOperationAction(ISD::SELECT, MVT::v4f32, Expand); setOperationAction(ISD::VSELECT, MVT::v4f32, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f32, Legal); setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f32, Expand); setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f32, Expand); setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f32, Expand); setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f32, Custom); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal); setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); setOperationAction(ISD::FP_TO_SINT , MVT::v4f32, Legal); setOperationAction(ISD::FP_TO_UINT , MVT::v4f32, Expand); setOperationAction(ISD::FNEG , MVT::v4f32, Legal); setOperationAction(ISD::FABS , MVT::v4f32, Legal); setOperationAction(ISD::FSIN , MVT::v4f32, Expand); setOperationAction(ISD::FCOS , MVT::v4f32, Expand); setOperationAction(ISD::FPOW , MVT::v4f32, Expand); setOperationAction(ISD::FLOG , MVT::v4f32, Expand); setOperationAction(ISD::FLOG2 , MVT::v4f32, Expand); setOperationAction(ISD::FLOG10 , MVT::v4f32, Expand); setOperationAction(ISD::FEXP , MVT::v4f32, Expand); setOperationAction(ISD::FEXP2 , MVT::v4f32, Expand); setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal); setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal); setIndexedLoadAction(ISD::PRE_INC, MVT::v4f32, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::v4f32, Legal); addRegisterClass(MVT::v4f32, &PPC::QSRCRegClass); setOperationAction(ISD::AND , MVT::v4i1, Legal); setOperationAction(ISD::OR , MVT::v4i1, Legal); setOperationAction(ISD::XOR , MVT::v4i1, Legal); if (!Subtarget.useCRBits()) setOperationAction(ISD::SELECT, MVT::v4i1, Expand); setOperationAction(ISD::VSELECT, MVT::v4i1, Legal); setOperationAction(ISD::LOAD , MVT::v4i1, Custom); setOperationAction(ISD::STORE , MVT::v4i1, Custom); setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4i1, Custom); setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4i1, Expand); setOperationAction(ISD::CONCAT_VECTORS , MVT::v4i1, Expand); setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4i1, Expand); setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4i1, Custom); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i1, Expand); setOperationAction(ISD::BUILD_VECTOR, MVT::v4i1, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::v4i1, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::v4i1, Custom); addRegisterClass(MVT::v4i1, &PPC::QBRCRegClass); setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal); setOperationAction(ISD::FCEIL, MVT::v4f64, Legal); setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal); setOperationAction(ISD::FROUND, MVT::v4f64, Legal); setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal); setOperationAction(ISD::FCEIL, MVT::v4f32, Legal); setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); setOperationAction(ISD::FROUND, MVT::v4f32, Legal); setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Expand); setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand); // These need to set FE_INEXACT, and so cannot be vectorized here. setOperationAction(ISD::FRINT, MVT::v4f64, Expand); setOperationAction(ISD::FRINT, MVT::v4f32, Expand); if (TM.Options.UnsafeFPMath) { setOperationAction(ISD::FDIV, MVT::v4f64, Legal); setOperationAction(ISD::FSQRT, MVT::v4f64, Legal); setOperationAction(ISD::FDIV, MVT::v4f32, Legal); setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); } else { setOperationAction(ISD::FDIV, MVT::v4f64, Expand); setOperationAction(ISD::FSQRT, MVT::v4f64, Expand); setOperationAction(ISD::FDIV, MVT::v4f32, Expand); setOperationAction(ISD::FSQRT, MVT::v4f32, Expand); } } if (Subtarget.has64BitSupport()) setOperationAction(ISD::PREFETCH, MVT::Other, Legal); setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, isPPC64 ? Legal : Custom); if (!isPPC64) { setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Expand); setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Expand); } setBooleanContents(ZeroOrOneBooleanContent); if (Subtarget.hasAltivec()) { // Altivec instructions set fields to all zeros or all ones. setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); } if (!isPPC64) { // These libcalls are not available in 32-bit. setLibcallName(RTLIB::SHL_I128, nullptr); setLibcallName(RTLIB::SRL_I128, nullptr); setLibcallName(RTLIB::SRA_I128, nullptr); } setStackPointerRegisterToSaveRestore(isPPC64 ? PPC::X1 : PPC::R1); // We have target-specific dag combine patterns for the following nodes: setTargetDAGCombine(ISD::ADD); setTargetDAGCombine(ISD::SHL); setTargetDAGCombine(ISD::SRA); setTargetDAGCombine(ISD::SRL); setTargetDAGCombine(ISD::MUL); setTargetDAGCombine(ISD::SINT_TO_FP); setTargetDAGCombine(ISD::BUILD_VECTOR); if (Subtarget.hasFPCVT()) setTargetDAGCombine(ISD::UINT_TO_FP); setTargetDAGCombine(ISD::LOAD); setTargetDAGCombine(ISD::STORE); setTargetDAGCombine(ISD::BR_CC); if (Subtarget.useCRBits()) setTargetDAGCombine(ISD::BRCOND); setTargetDAGCombine(ISD::BSWAP); setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN); setTargetDAGCombine(ISD::INTRINSIC_VOID); setTargetDAGCombine(ISD::SIGN_EXTEND); setTargetDAGCombine(ISD::ZERO_EXTEND); setTargetDAGCombine(ISD::ANY_EXTEND); setTargetDAGCombine(ISD::TRUNCATE); if (Subtarget.useCRBits()) { setTargetDAGCombine(ISD::TRUNCATE); setTargetDAGCombine(ISD::SETCC); setTargetDAGCombine(ISD::SELECT_CC); } // Use reciprocal estimates. if (TM.Options.UnsafeFPMath) { setTargetDAGCombine(ISD::FDIV); setTargetDAGCombine(ISD::FSQRT); } if (Subtarget.hasP9Altivec()) { setTargetDAGCombine(ISD::ABS); setTargetDAGCombine(ISD::VSELECT); } // Darwin long double math library functions have $LDBL128 appended. if (Subtarget.isDarwin()) { setLibcallName(RTLIB::COS_PPCF128, "cosl$LDBL128"); setLibcallName(RTLIB::POW_PPCF128, "powl$LDBL128"); setLibcallName(RTLIB::REM_PPCF128, "fmodl$LDBL128"); setLibcallName(RTLIB::SIN_PPCF128, "sinl$LDBL128"); setLibcallName(RTLIB::SQRT_PPCF128, "sqrtl$LDBL128"); setLibcallName(RTLIB::LOG_PPCF128, "logl$LDBL128"); setLibcallName(RTLIB::LOG2_PPCF128, "log2l$LDBL128"); setLibcallName(RTLIB::LOG10_PPCF128, "log10l$LDBL128"); setLibcallName(RTLIB::EXP_PPCF128, "expl$LDBL128"); setLibcallName(RTLIB::EXP2_PPCF128, "exp2l$LDBL128"); } if (EnableQuadPrecision) { setLibcallName(RTLIB::LOG_F128, "logf128"); setLibcallName(RTLIB::LOG2_F128, "log2f128"); setLibcallName(RTLIB::LOG10_F128, "log10f128"); setLibcallName(RTLIB::EXP_F128, "expf128"); setLibcallName(RTLIB::EXP2_F128, "exp2f128"); setLibcallName(RTLIB::SIN_F128, "sinf128"); setLibcallName(RTLIB::COS_F128, "cosf128"); setLibcallName(RTLIB::POW_F128, "powf128"); setLibcallName(RTLIB::FMIN_F128, "fminf128"); setLibcallName(RTLIB::FMAX_F128, "fmaxf128"); setLibcallName(RTLIB::POWI_F128, "__powikf2"); setLibcallName(RTLIB::REM_F128, "fmodf128"); } // With 32 condition bits, we don't need to sink (and duplicate) compares // aggressively in CodeGenPrep. if (Subtarget.useCRBits()) { setHasMultipleConditionRegisters(); setJumpIsExpensive(); } setMinFunctionAlignment(2); if (Subtarget.isDarwin()) setPrefFunctionAlignment(4); switch (Subtarget.getDarwinDirective()) { default: break; case PPC::DIR_970: case PPC::DIR_A2: case PPC::DIR_E500: case PPC::DIR_E500mc: case PPC::DIR_E5500: case PPC::DIR_PWR4: case PPC::DIR_PWR5: case PPC::DIR_PWR5X: case PPC::DIR_PWR6: case PPC::DIR_PWR6X: case PPC::DIR_PWR7: case PPC::DIR_PWR8: case PPC::DIR_PWR9: setPrefFunctionAlignment(4); setPrefLoopAlignment(4); break; } if (Subtarget.enableMachineScheduler()) setSchedulingPreference(Sched::Source); else setSchedulingPreference(Sched::Hybrid); computeRegisterProperties(STI.getRegisterInfo()); // The Freescale cores do better with aggressive inlining of memcpy and // friends. GCC uses same threshold of 128 bytes (= 32 word stores). if (Subtarget.getDarwinDirective() == PPC::DIR_E500mc || Subtarget.getDarwinDirective() == PPC::DIR_E5500) { MaxStoresPerMemset = 32; MaxStoresPerMemsetOptSize = 16; MaxStoresPerMemcpy = 32; MaxStoresPerMemcpyOptSize = 8; MaxStoresPerMemmove = 32; MaxStoresPerMemmoveOptSize = 8; } else if (Subtarget.getDarwinDirective() == PPC::DIR_A2) { // The A2 also benefits from (very) aggressive inlining of memcpy and // friends. The overhead of a the function call, even when warm, can be // over one hundred cycles. MaxStoresPerMemset = 128; MaxStoresPerMemcpy = 128; MaxStoresPerMemmove = 128; MaxLoadsPerMemcmp = 128; } else { MaxLoadsPerMemcmp = 8; MaxLoadsPerMemcmpOptSize = 4; } } /// getMaxByValAlign - Helper for getByValTypeAlignment to determine /// the desired ByVal argument alignment. static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign, unsigned MaxMaxAlign) { if (MaxAlign == MaxMaxAlign) return; if (VectorType *VTy = dyn_cast(Ty)) { if (MaxMaxAlign >= 32 && VTy->getBitWidth() >= 256) MaxAlign = 32; else if (VTy->getBitWidth() >= 128 && MaxAlign < 16) MaxAlign = 16; } else if (ArrayType *ATy = dyn_cast(Ty)) { unsigned EltAlign = 0; getMaxByValAlign(ATy->getElementType(), EltAlign, MaxMaxAlign); if (EltAlign > MaxAlign) MaxAlign = EltAlign; } else if (StructType *STy = dyn_cast(Ty)) { for (auto *EltTy : STy->elements()) { unsigned EltAlign = 0; getMaxByValAlign(EltTy, EltAlign, MaxMaxAlign); if (EltAlign > MaxAlign) MaxAlign = EltAlign; if (MaxAlign == MaxMaxAlign) break; } } } /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate /// function arguments in the caller parameter area. unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty, const DataLayout &DL) const { // Darwin passes everything on 4 byte boundary. if (Subtarget.isDarwin()) return 4; // 16byte and wider vectors are passed on 16byte boundary. // The rest is 8 on PPC64 and 4 on PPC32 boundary. unsigned Align = Subtarget.isPPC64() ? 8 : 4; if (Subtarget.hasAltivec() || Subtarget.hasQPX()) getMaxByValAlign(Ty, Align, Subtarget.hasQPX() ? 32 : 16); return Align; } bool PPCTargetLowering::useSoftFloat() const { return Subtarget.useSoftFloat(); } bool PPCTargetLowering::hasSPE() const { return Subtarget.hasSPE(); } bool PPCTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const { return VT.isScalarInteger(); } const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { switch ((PPCISD::NodeType)Opcode) { case PPCISD::FIRST_NUMBER: break; case PPCISD::FSEL: return "PPCISD::FSEL"; case PPCISD::FCFID: return "PPCISD::FCFID"; case PPCISD::FCFIDU: return "PPCISD::FCFIDU"; case PPCISD::FCFIDS: return "PPCISD::FCFIDS"; case PPCISD::FCFIDUS: return "PPCISD::FCFIDUS"; case PPCISD::FCTIDZ: return "PPCISD::FCTIDZ"; case PPCISD::FCTIWZ: return "PPCISD::FCTIWZ"; case PPCISD::FCTIDUZ: return "PPCISD::FCTIDUZ"; case PPCISD::FCTIWUZ: return "PPCISD::FCTIWUZ"; case PPCISD::FP_TO_UINT_IN_VSR: return "PPCISD::FP_TO_UINT_IN_VSR,"; case PPCISD::FP_TO_SINT_IN_VSR: return "PPCISD::FP_TO_SINT_IN_VSR"; case PPCISD::FRE: return "PPCISD::FRE"; case PPCISD::FRSQRTE: return "PPCISD::FRSQRTE"; case PPCISD::STFIWX: return "PPCISD::STFIWX"; case PPCISD::VMADDFP: return "PPCISD::VMADDFP"; case PPCISD::VNMSUBFP: return "PPCISD::VNMSUBFP"; case PPCISD::VPERM: return "PPCISD::VPERM"; case PPCISD::XXSPLT: return "PPCISD::XXSPLT"; case PPCISD::VECINSERT: return "PPCISD::VECINSERT"; case PPCISD::XXREVERSE: return "PPCISD::XXREVERSE"; case PPCISD::XXPERMDI: return "PPCISD::XXPERMDI"; case PPCISD::VECSHL: return "PPCISD::VECSHL"; case PPCISD::CMPB: return "PPCISD::CMPB"; case PPCISD::Hi: return "PPCISD::Hi"; case PPCISD::Lo: return "PPCISD::Lo"; case PPCISD::TOC_ENTRY: return "PPCISD::TOC_ENTRY"; case PPCISD::ATOMIC_CMP_SWAP_8: return "PPCISD::ATOMIC_CMP_SWAP_8"; case PPCISD::ATOMIC_CMP_SWAP_16: return "PPCISD::ATOMIC_CMP_SWAP_16"; case PPCISD::DYNALLOC: return "PPCISD::DYNALLOC"; case PPCISD::DYNAREAOFFSET: return "PPCISD::DYNAREAOFFSET"; case PPCISD::GlobalBaseReg: return "PPCISD::GlobalBaseReg"; case PPCISD::SRL: return "PPCISD::SRL"; case PPCISD::SRA: return "PPCISD::SRA"; case PPCISD::SHL: return "PPCISD::SHL"; case PPCISD::SRA_ADDZE: return "PPCISD::SRA_ADDZE"; case PPCISD::CALL: return "PPCISD::CALL"; case PPCISD::CALL_NOP: return "PPCISD::CALL_NOP"; case PPCISD::MTCTR: return "PPCISD::MTCTR"; case PPCISD::BCTRL: return "PPCISD::BCTRL"; case PPCISD::BCTRL_LOAD_TOC: return "PPCISD::BCTRL_LOAD_TOC"; case PPCISD::RET_FLAG: return "PPCISD::RET_FLAG"; case PPCISD::READ_TIME_BASE: return "PPCISD::READ_TIME_BASE"; case PPCISD::EH_SJLJ_SETJMP: return "PPCISD::EH_SJLJ_SETJMP"; case PPCISD::EH_SJLJ_LONGJMP: return "PPCISD::EH_SJLJ_LONGJMP"; case PPCISD::MFOCRF: return "PPCISD::MFOCRF"; case PPCISD::MFVSR: return "PPCISD::MFVSR"; case PPCISD::MTVSRA: return "PPCISD::MTVSRA"; case PPCISD::MTVSRZ: return "PPCISD::MTVSRZ"; case PPCISD::SINT_VEC_TO_FP: return "PPCISD::SINT_VEC_TO_FP"; case PPCISD::UINT_VEC_TO_FP: return "PPCISD::UINT_VEC_TO_FP"; case PPCISD::ANDIo_1_EQ_BIT: return "PPCISD::ANDIo_1_EQ_BIT"; case PPCISD::ANDIo_1_GT_BIT: return "PPCISD::ANDIo_1_GT_BIT"; case PPCISD::VCMP: return "PPCISD::VCMP"; case PPCISD::VCMPo: return "PPCISD::VCMPo"; case PPCISD::LBRX: return "PPCISD::LBRX"; case PPCISD::STBRX: return "PPCISD::STBRX"; case PPCISD::LFIWAX: return "PPCISD::LFIWAX"; case PPCISD::LFIWZX: return "PPCISD::LFIWZX"; case PPCISD::LXSIZX: return "PPCISD::LXSIZX"; case PPCISD::STXSIX: return "PPCISD::STXSIX"; case PPCISD::VEXTS: return "PPCISD::VEXTS"; case PPCISD::SExtVElems: return "PPCISD::SExtVElems"; case PPCISD::LXVD2X: return "PPCISD::LXVD2X"; case PPCISD::STXVD2X: return "PPCISD::STXVD2X"; case PPCISD::ST_VSR_SCAL_INT: return "PPCISD::ST_VSR_SCAL_INT"; case PPCISD::COND_BRANCH: return "PPCISD::COND_BRANCH"; case PPCISD::BDNZ: return "PPCISD::BDNZ"; case PPCISD::BDZ: return "PPCISD::BDZ"; case PPCISD::MFFS: return "PPCISD::MFFS"; case PPCISD::FADDRTZ: return "PPCISD::FADDRTZ"; case PPCISD::TC_RETURN: return "PPCISD::TC_RETURN"; case PPCISD::CR6SET: return "PPCISD::CR6SET"; case PPCISD::CR6UNSET: return "PPCISD::CR6UNSET"; case PPCISD::PPC32_GOT: return "PPCISD::PPC32_GOT"; case PPCISD::PPC32_PICGOT: return "PPCISD::PPC32_PICGOT"; case PPCISD::ADDIS_GOT_TPREL_HA: return "PPCISD::ADDIS_GOT_TPREL_HA"; case PPCISD::LD_GOT_TPREL_L: return "PPCISD::LD_GOT_TPREL_L"; case PPCISD::ADD_TLS: return "PPCISD::ADD_TLS"; case PPCISD::ADDIS_TLSGD_HA: return "PPCISD::ADDIS_TLSGD_HA"; case PPCISD::ADDI_TLSGD_L: return "PPCISD::ADDI_TLSGD_L"; case PPCISD::GET_TLS_ADDR: return "PPCISD::GET_TLS_ADDR"; case PPCISD::ADDI_TLSGD_L_ADDR: return "PPCISD::ADDI_TLSGD_L_ADDR"; case PPCISD::ADDIS_TLSLD_HA: return "PPCISD::ADDIS_TLSLD_HA"; case PPCISD::ADDI_TLSLD_L: return "PPCISD::ADDI_TLSLD_L"; case PPCISD::GET_TLSLD_ADDR: return "PPCISD::GET_TLSLD_ADDR"; case PPCISD::ADDI_TLSLD_L_ADDR: return "PPCISD::ADDI_TLSLD_L_ADDR"; case PPCISD::ADDIS_DTPREL_HA: return "PPCISD::ADDIS_DTPREL_HA"; case PPCISD::ADDI_DTPREL_L: return "PPCISD::ADDI_DTPREL_L"; case PPCISD::VADD_SPLAT: return "PPCISD::VADD_SPLAT"; case PPCISD::SC: return "PPCISD::SC"; case PPCISD::CLRBHRB: return "PPCISD::CLRBHRB"; case PPCISD::MFBHRBE: return "PPCISD::MFBHRBE"; case PPCISD::RFEBB: return "PPCISD::RFEBB"; case PPCISD::XXSWAPD: return "PPCISD::XXSWAPD"; case PPCISD::SWAP_NO_CHAIN: return "PPCISD::SWAP_NO_CHAIN"; case PPCISD::VABSD: return "PPCISD::VABSD"; case PPCISD::QVFPERM: return "PPCISD::QVFPERM"; case PPCISD::QVGPCI: return "PPCISD::QVGPCI"; case PPCISD::QVALIGNI: return "PPCISD::QVALIGNI"; case PPCISD::QVESPLATI: return "PPCISD::QVESPLATI"; case PPCISD::QBFLT: return "PPCISD::QBFLT"; case PPCISD::QVLFSb: return "PPCISD::QVLFSb"; case PPCISD::BUILD_FP128: return "PPCISD::BUILD_FP128"; case PPCISD::BUILD_SPE64: return "PPCISD::BUILD_SPE64"; case PPCISD::EXTRACT_SPE: return "PPCISD::EXTRACT_SPE"; case PPCISD::EXTSWSLI: return "PPCISD::EXTSWSLI"; case PPCISD::LD_VSX_LH: return "PPCISD::LD_VSX_LH"; case PPCISD::FP_EXTEND_LH: return "PPCISD::FP_EXTEND_LH"; } return nullptr; } EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C, EVT VT) const { if (!VT.isVector()) return Subtarget.useCRBits() ? MVT::i1 : MVT::i32; if (Subtarget.hasQPX()) return EVT::getVectorVT(C, MVT::i1, VT.getVectorNumElements()); return VT.changeVectorElementTypeToInteger(); } bool PPCTargetLowering::enableAggressiveFMAFusion(EVT VT) const { assert(VT.isFloatingPoint() && "Non-floating-point FMA?"); return true; } //===----------------------------------------------------------------------===// // Node matching predicates, for use by the tblgen matching code. //===----------------------------------------------------------------------===// /// isFloatingPointZero - Return true if this is 0.0 or -0.0. static bool isFloatingPointZero(SDValue Op) { if (ConstantFPSDNode *CFP = dyn_cast(Op)) return CFP->getValueAPF().isZero(); else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) { // Maybe this has already been legalized into the constant pool? if (ConstantPoolSDNode *CP = dyn_cast(Op.getOperand(1))) if (const ConstantFP *CFP = dyn_cast(CP->getConstVal())) return CFP->getValueAPF().isZero(); } return false; } /// isConstantOrUndef - Op is either an undef node or a ConstantSDNode. Return /// true if Op is undef or if it matches the specified value. static bool isConstantOrUndef(int Op, int Val) { return Op < 0 || Op == Val; } /// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a /// VPKUHUM instruction. /// The ShuffleKind distinguishes between big-endian operations with /// two different inputs (0), either-endian operations with two identical /// inputs (1), and little-endian operations with two different inputs (2). /// For the latter, the input operands are swapped (see PPCInstrAltivec.td). bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, SelectionDAG &DAG) { bool IsLE = DAG.getDataLayout().isLittleEndian(); if (ShuffleKind == 0) { if (IsLE) return false; for (unsigned i = 0; i != 16; ++i) if (!isConstantOrUndef(N->getMaskElt(i), i*2+1)) return false; } else if (ShuffleKind == 2) { if (!IsLE) return false; for (unsigned i = 0; i != 16; ++i) if (!isConstantOrUndef(N->getMaskElt(i), i*2)) return false; } else if (ShuffleKind == 1) { unsigned j = IsLE ? 0 : 1; for (unsigned i = 0; i != 8; ++i) if (!isConstantOrUndef(N->getMaskElt(i), i*2+j) || !isConstantOrUndef(N->getMaskElt(i+8), i*2+j)) return false; } return true; } /// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a /// VPKUWUM instruction. /// The ShuffleKind distinguishes between big-endian operations with /// two different inputs (0), either-endian operations with two identical /// inputs (1), and little-endian operations with two different inputs (2). /// For the latter, the input operands are swapped (see PPCInstrAltivec.td). bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, SelectionDAG &DAG) { bool IsLE = DAG.getDataLayout().isLittleEndian(); if (ShuffleKind == 0) { if (IsLE) return false; for (unsigned i = 0; i != 16; i += 2) if (!isConstantOrUndef(N->getMaskElt(i ), i*2+2) || !isConstantOrUndef(N->getMaskElt(i+1), i*2+3)) return false; } else if (ShuffleKind == 2) { if (!IsLE) return false; for (unsigned i = 0; i != 16; i += 2) if (!isConstantOrUndef(N->getMaskElt(i ), i*2) || !isConstantOrUndef(N->getMaskElt(i+1), i*2+1)) return false; } else if (ShuffleKind == 1) { unsigned j = IsLE ? 0 : 2; for (unsigned i = 0; i != 8; i += 2) if (!isConstantOrUndef(N->getMaskElt(i ), i*2+j) || !isConstantOrUndef(N->getMaskElt(i+1), i*2+j+1) || !isConstantOrUndef(N->getMaskElt(i+8), i*2+j) || !isConstantOrUndef(N->getMaskElt(i+9), i*2+j+1)) return false; } return true; } /// isVPKUDUMShuffleMask - Return true if this is the shuffle mask for a /// VPKUDUM instruction, AND the VPKUDUM instruction exists for the /// current subtarget. /// /// The ShuffleKind distinguishes between big-endian operations with /// two different inputs (0), either-endian operations with two identical /// inputs (1), and little-endian operations with two different inputs (2). /// For the latter, the input operands are swapped (see PPCInstrAltivec.td). bool PPC::isVPKUDUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, SelectionDAG &DAG) { const PPCSubtarget& Subtarget = static_cast(DAG.getSubtarget()); if (!Subtarget.hasP8Vector()) return false; bool IsLE = DAG.getDataLayout().isLittleEndian(); if (ShuffleKind == 0) { if (IsLE) return false; for (unsigned i = 0; i != 16; i += 4) if (!isConstantOrUndef(N->getMaskElt(i ), i*2+4) || !isConstantOrUndef(N->getMaskElt(i+1), i*2+5) || !isConstantOrUndef(N->getMaskElt(i+2), i*2+6) || !isConstantOrUndef(N->getMaskElt(i+3), i*2+7)) return false; } else if (ShuffleKind == 2) { if (!IsLE) return false; for (unsigned i = 0; i != 16; i += 4) if (!isConstantOrUndef(N->getMaskElt(i ), i*2) || !isConstantOrUndef(N->getMaskElt(i+1), i*2+1) || !isConstantOrUndef(N->getMaskElt(i+2), i*2+2) || !isConstantOrUndef(N->getMaskElt(i+3), i*2+3)) return false; } else if (ShuffleKind == 1) { unsigned j = IsLE ? 0 : 4; for (unsigned i = 0; i != 8; i += 4) if (!isConstantOrUndef(N->getMaskElt(i ), i*2+j) || !isConstantOrUndef(N->getMaskElt(i+1), i*2+j+1) || !isConstantOrUndef(N->getMaskElt(i+2), i*2+j+2) || !isConstantOrUndef(N->getMaskElt(i+3), i*2+j+3) || !isConstantOrUndef(N->getMaskElt(i+8), i*2+j) || !isConstantOrUndef(N->getMaskElt(i+9), i*2+j+1) || !isConstantOrUndef(N->getMaskElt(i+10), i*2+j+2) || !isConstantOrUndef(N->getMaskElt(i+11), i*2+j+3)) return false; } return true; } /// isVMerge - Common function, used to match vmrg* shuffles. /// static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize, unsigned LHSStart, unsigned RHSStart) { if (N->getValueType(0) != MVT::v16i8) return false; assert((UnitSize == 1 || UnitSize == 2 || UnitSize == 4) && "Unsupported merge size!"); for (unsigned i = 0; i != 8/UnitSize; ++i) // Step over units for (unsigned j = 0; j != UnitSize; ++j) { // Step over bytes within unit if (!isConstantOrUndef(N->getMaskElt(i*UnitSize*2+j), LHSStart+j+i*UnitSize) || !isConstantOrUndef(N->getMaskElt(i*UnitSize*2+UnitSize+j), RHSStart+j+i*UnitSize)) return false; } return true; } /// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for /// a VMRGL* instruction with the specified unit size (1,2 or 4 bytes). /// The ShuffleKind distinguishes between big-endian merges with two /// different inputs (0), either-endian merges with two identical inputs (1), /// and little-endian merges with two different inputs (2). For the latter, /// the input operands are swapped (see PPCInstrAltivec.td). bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, unsigned ShuffleKind, SelectionDAG &DAG) { if (DAG.getDataLayout().isLittleEndian()) { if (ShuffleKind == 1) // unary return isVMerge(N, UnitSize, 0, 0); else if (ShuffleKind == 2) // swapped return isVMerge(N, UnitSize, 0, 16); else return false; } else { if (ShuffleKind == 1) // unary return isVMerge(N, UnitSize, 8, 8); else if (ShuffleKind == 0) // normal return isVMerge(N, UnitSize, 8, 24); else return false; } } /// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for /// a VMRGH* instruction with the specified unit size (1,2 or 4 bytes). /// The ShuffleKind distinguishes between big-endian merges with two /// different inputs (0), either-endian merges with two identical inputs (1), /// and little-endian merges with two different inputs (2). For the latter, /// the input operands are swapped (see PPCInstrAltivec.td). bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, unsigned ShuffleKind, SelectionDAG &DAG) { if (DAG.getDataLayout().isLittleEndian()) { if (ShuffleKind == 1) // unary return isVMerge(N, UnitSize, 8, 8); else if (ShuffleKind == 2) // swapped return isVMerge(N, UnitSize, 8, 24); else return false; } else { if (ShuffleKind == 1) // unary return isVMerge(N, UnitSize, 0, 0); else if (ShuffleKind == 0) // normal return isVMerge(N, UnitSize, 0, 16); else return false; } } /** * Common function used to match vmrgew and vmrgow shuffles * * The indexOffset determines whether to look for even or odd words in * the shuffle mask. This is based on the of the endianness of the target * machine. * - Little Endian: * - Use offset of 0 to check for odd elements * - Use offset of 4 to check for even elements * - Big Endian: * - Use offset of 0 to check for even elements * - Use offset of 4 to check for odd elements * A detailed description of the vector element ordering for little endian and * big endian can be found at * http://www.ibm.com/developerworks/library/l-ibm-xl-c-cpp-compiler/index.html * Targeting your applications - what little endian and big endian IBM XL C/C++ * compiler differences mean to you * * The mask to the shuffle vector instruction specifies the indices of the * elements from the two input vectors to place in the result. The elements are * numbered in array-access order, starting with the first vector. These vectors * are always of type v16i8, thus each vector will contain 16 elements of size * 8. More info on the shuffle vector can be found in the * http://llvm.org/docs/LangRef.html#shufflevector-instruction * Language Reference. * * The RHSStartValue indicates whether the same input vectors are used (unary) * or two different input vectors are used, based on the following: * - If the instruction uses the same vector for both inputs, the range of the * indices will be 0 to 15. In this case, the RHSStart value passed should * be 0. * - If the instruction has two different vectors then the range of the * indices will be 0 to 31. In this case, the RHSStart value passed should * be 16 (indices 0-15 specify elements in the first vector while indices 16 * to 31 specify elements in the second vector). * * \param[in] N The shuffle vector SD Node to analyze * \param[in] IndexOffset Specifies whether to look for even or odd elements * \param[in] RHSStartValue Specifies the starting index for the righthand input * vector to the shuffle_vector instruction * \return true iff this shuffle vector represents an even or odd word merge */ static bool isVMerge(ShuffleVectorSDNode *N, unsigned IndexOffset, unsigned RHSStartValue) { if (N->getValueType(0) != MVT::v16i8) return false; for (unsigned i = 0; i < 2; ++i) for (unsigned j = 0; j < 4; ++j) if (!isConstantOrUndef(N->getMaskElt(i*4+j), i*RHSStartValue+j+IndexOffset) || !isConstantOrUndef(N->getMaskElt(i*4+j+8), i*RHSStartValue+j+IndexOffset+8)) return false; return true; } /** * Determine if the specified shuffle mask is suitable for the vmrgew or * vmrgow instructions. * * \param[in] N The shuffle vector SD Node to analyze * \param[in] CheckEven Check for an even merge (true) or an odd merge (false) * \param[in] ShuffleKind Identify the type of merge: * - 0 = big-endian merge with two different inputs; * - 1 = either-endian merge with two identical inputs; * - 2 = little-endian merge with two different inputs (inputs are swapped for * little-endian merges). * \param[in] DAG The current SelectionDAG * \return true iff this shuffle mask */ bool PPC::isVMRGEOShuffleMask(ShuffleVectorSDNode *N, bool CheckEven, unsigned ShuffleKind, SelectionDAG &DAG) { if (DAG.getDataLayout().isLittleEndian()) { unsigned indexOffset = CheckEven ? 4 : 0; if (ShuffleKind == 1) // Unary return isVMerge(N, indexOffset, 0); else if (ShuffleKind == 2) // swapped return isVMerge(N, indexOffset, 16); else return false; } else { unsigned indexOffset = CheckEven ? 0 : 4; if (ShuffleKind == 1) // Unary return isVMerge(N, indexOffset, 0); else if (ShuffleKind == 0) // Normal return isVMerge(N, indexOffset, 16); else return false; } return false; } /// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift /// amount, otherwise return -1. /// The ShuffleKind distinguishes between big-endian operations with two /// different inputs (0), either-endian operations with two identical inputs /// (1), and little-endian operations with two different inputs (2). For the /// latter, the input operands are swapped (see PPCInstrAltivec.td). int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind, SelectionDAG &DAG) { if (N->getValueType(0) != MVT::v16i8) return -1; ShuffleVectorSDNode *SVOp = cast(N); // Find the first non-undef value in the shuffle mask. unsigned i; for (i = 0; i != 16 && SVOp->getMaskElt(i) < 0; ++i) /*search*/; if (i == 16) return -1; // all undef. // Otherwise, check to see if the rest of the elements are consecutively // numbered from this value. unsigned ShiftAmt = SVOp->getMaskElt(i); if (ShiftAmt < i) return -1; ShiftAmt -= i; bool isLE = DAG.getDataLayout().isLittleEndian(); if ((ShuffleKind == 0 && !isLE) || (ShuffleKind == 2 && isLE)) { // Check the rest of the elements to see if they are consecutive. for (++i; i != 16; ++i) if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i)) return -1; } else if (ShuffleKind == 1) { // Check the rest of the elements to see if they are consecutive. for (++i; i != 16; ++i) if (!isConstantOrUndef(SVOp->getMaskElt(i), (ShiftAmt+i) & 15)) return -1; } else return -1; if (isLE) ShiftAmt = 16 - ShiftAmt; return ShiftAmt; } /// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand /// specifies a splat of a single element that is suitable for input to /// VSPLTB/VSPLTH/VSPLTW. bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) { assert(N->getValueType(0) == MVT::v16i8 && (EltSize == 1 || EltSize == 2 || EltSize == 4)); // The consecutive indices need to specify an element, not part of two // different elements. So abandon ship early if this isn't the case. if (N->getMaskElt(0) % EltSize != 0) return false; // This is a splat operation if each element of the permute is the same, and // if the value doesn't reference the second vector. unsigned ElementBase = N->getMaskElt(0); // FIXME: Handle UNDEF elements too! if (ElementBase >= 16) return false; // Check that the indices are consecutive, in the case of a multi-byte element // splatted with a v16i8 mask. for (unsigned i = 1; i != EltSize; ++i) if (N->getMaskElt(i) < 0 || N->getMaskElt(i) != (int)(i+ElementBase)) return false; for (unsigned i = EltSize, e = 16; i != e; i += EltSize) { if (N->getMaskElt(i) < 0) continue; for (unsigned j = 0; j != EltSize; ++j) if (N->getMaskElt(i+j) != N->getMaskElt(j)) return false; } return true; } /// Check that the mask is shuffling N byte elements. Within each N byte /// element of the mask, the indices could be either in increasing or /// decreasing order as long as they are consecutive. /// \param[in] N the shuffle vector SD Node to analyze /// \param[in] Width the element width in bytes, could be 2/4/8/16 (HalfWord/ /// Word/DoubleWord/QuadWord). /// \param[in] StepLen the delta indices number among the N byte element, if /// the mask is in increasing/decreasing order then it is 1/-1. /// \return true iff the mask is shuffling N byte elements. static bool isNByteElemShuffleMask(ShuffleVectorSDNode *N, unsigned Width, int StepLen) { assert((Width == 2 || Width == 4 || Width == 8 || Width == 16) && "Unexpected element width."); assert((StepLen == 1 || StepLen == -1) && "Unexpected element width."); unsigned NumOfElem = 16 / Width; unsigned MaskVal[16]; // Width is never greater than 16 for (unsigned i = 0; i < NumOfElem; ++i) { MaskVal[0] = N->getMaskElt(i * Width); if ((StepLen == 1) && (MaskVal[0] % Width)) { return false; } else if ((StepLen == -1) && ((MaskVal[0] + 1) % Width)) { return false; } for (unsigned int j = 1; j < Width; ++j) { MaskVal[j] = N->getMaskElt(i * Width + j); if (MaskVal[j] != MaskVal[j-1] + StepLen) { return false; } } } return true; } bool PPC::isXXINSERTWMask(ShuffleVectorSDNode *N, unsigned &ShiftElts, unsigned &InsertAtByte, bool &Swap, bool IsLE) { if (!isNByteElemShuffleMask(N, 4, 1)) return false; // Now we look at mask elements 0,4,8,12 unsigned M0 = N->getMaskElt(0) / 4; unsigned M1 = N->getMaskElt(4) / 4; unsigned M2 = N->getMaskElt(8) / 4; unsigned M3 = N->getMaskElt(12) / 4; unsigned LittleEndianShifts[] = { 2, 1, 0, 3 }; unsigned BigEndianShifts[] = { 3, 0, 1, 2 }; // Below, let H and L be arbitrary elements of the shuffle mask // where H is in the range [4,7] and L is in the range [0,3]. // H, 1, 2, 3 or L, 5, 6, 7 if ((M0 > 3 && M1 == 1 && M2 == 2 && M3 == 3) || (M0 < 4 && M1 == 5 && M2 == 6 && M3 == 7)) { ShiftElts = IsLE ? LittleEndianShifts[M0 & 0x3] : BigEndianShifts[M0 & 0x3]; InsertAtByte = IsLE ? 12 : 0; Swap = M0 < 4; return true; } // 0, H, 2, 3 or 4, L, 6, 7 if ((M1 > 3 && M0 == 0 && M2 == 2 && M3 == 3) || (M1 < 4 && M0 == 4 && M2 == 6 && M3 == 7)) { ShiftElts = IsLE ? LittleEndianShifts[M1 & 0x3] : BigEndianShifts[M1 & 0x3]; InsertAtByte = IsLE ? 8 : 4; Swap = M1 < 4; return true; } // 0, 1, H, 3 or 4, 5, L, 7 if ((M2 > 3 && M0 == 0 && M1 == 1 && M3 == 3) || (M2 < 4 && M0 == 4 && M1 == 5 && M3 == 7)) { ShiftElts = IsLE ? LittleEndianShifts[M2 & 0x3] : BigEndianShifts[M2 & 0x3]; InsertAtByte = IsLE ? 4 : 8; Swap = M2 < 4; return true; } // 0, 1, 2, H or 4, 5, 6, L if ((M3 > 3 && M0 == 0 && M1 == 1 && M2 == 2) || (M3 < 4 && M0 == 4 && M1 == 5 && M2 == 6)) { ShiftElts = IsLE ? LittleEndianShifts[M3 & 0x3] : BigEndianShifts[M3 & 0x3]; InsertAtByte = IsLE ? 0 : 12; Swap = M3 < 4; return true; } // If both vector operands for the shuffle are the same vector, the mask will // contain only elements from the first one and the second one will be undef. if (N->getOperand(1).isUndef()) { ShiftElts = 0; Swap = true; unsigned XXINSERTWSrcElem = IsLE ? 2 : 1; if (M0 == XXINSERTWSrcElem && M1 == 1 && M2 == 2 && M3 == 3) { InsertAtByte = IsLE ? 12 : 0; return true; } if (M0 == 0 && M1 == XXINSERTWSrcElem && M2 == 2 && M3 == 3) { InsertAtByte = IsLE ? 8 : 4; return true; } if (M0 == 0 && M1 == 1 && M2 == XXINSERTWSrcElem && M3 == 3) { InsertAtByte = IsLE ? 4 : 8; return true; } if (M0 == 0 && M1 == 1 && M2 == 2 && M3 == XXINSERTWSrcElem) { InsertAtByte = IsLE ? 0 : 12; return true; } } return false; } bool PPC::isXXSLDWIShuffleMask(ShuffleVectorSDNode *N, unsigned &ShiftElts, bool &Swap, bool IsLE) { assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8"); // Ensure each byte index of the word is consecutive. if (!isNByteElemShuffleMask(N, 4, 1)) return false; // Now we look at mask elements 0,4,8,12, which are the beginning of words. unsigned M0 = N->getMaskElt(0) / 4; unsigned M1 = N->getMaskElt(4) / 4; unsigned M2 = N->getMaskElt(8) / 4; unsigned M3 = N->getMaskElt(12) / 4; // If both vector operands for the shuffle are the same vector, the mask will // contain only elements from the first one and the second one will be undef. if (N->getOperand(1).isUndef()) { assert(M0 < 4 && "Indexing into an undef vector?"); if (M1 != (M0 + 1) % 4 || M2 != (M1 + 1) % 4 || M3 != (M2 + 1) % 4) return false; ShiftElts = IsLE ? (4 - M0) % 4 : M0; Swap = false; return true; } // Ensure each word index of the ShuffleVector Mask is consecutive. if (M1 != (M0 + 1) % 8 || M2 != (M1 + 1) % 8 || M3 != (M2 + 1) % 8) return false; if (IsLE) { if (M0 == 0 || M0 == 7 || M0 == 6 || M0 == 5) { // Input vectors don't need to be swapped if the leading element // of the result is one of the 3 left elements of the second vector // (or if there is no shift to be done at all). Swap = false; ShiftElts = (8 - M0) % 8; } else if (M0 == 4 || M0 == 3 || M0 == 2 || M0 == 1) { // Input vectors need to be swapped if the leading element // of the result is one of the 3 left elements of the first vector // (or if we're shifting by 4 - thereby simply swapping the vectors). Swap = true; ShiftElts = (4 - M0) % 4; } return true; } else { // BE if (M0 == 0 || M0 == 1 || M0 == 2 || M0 == 3) { // Input vectors don't need to be swapped if the leading element // of the result is one of the 4 elements of the first vector. Swap = false; ShiftElts = M0; } else if (M0 == 4 || M0 == 5 || M0 == 6 || M0 == 7) { // Input vectors need to be swapped if the leading element // of the result is one of the 4 elements of the right vector. Swap = true; ShiftElts = M0 - 4; } return true; } } bool static isXXBRShuffleMaskHelper(ShuffleVectorSDNode *N, int Width) { assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8"); if (!isNByteElemShuffleMask(N, Width, -1)) return false; for (int i = 0; i < 16; i += Width) if (N->getMaskElt(i) != i + Width - 1) return false; return true; } bool PPC::isXXBRHShuffleMask(ShuffleVectorSDNode *N) { return isXXBRShuffleMaskHelper(N, 2); } bool PPC::isXXBRWShuffleMask(ShuffleVectorSDNode *N) { return isXXBRShuffleMaskHelper(N, 4); } bool PPC::isXXBRDShuffleMask(ShuffleVectorSDNode *N) { return isXXBRShuffleMaskHelper(N, 8); } bool PPC::isXXBRQShuffleMask(ShuffleVectorSDNode *N) { return isXXBRShuffleMaskHelper(N, 16); } /// Can node \p N be lowered to an XXPERMDI instruction? If so, set \p Swap /// if the inputs to the instruction should be swapped and set \p DM to the /// value for the immediate. /// Specifically, set \p Swap to true only if \p N can be lowered to XXPERMDI /// AND element 0 of the result comes from the first input (LE) or second input /// (BE). Set \p DM to the calculated result (0-3) only if \p N can be lowered. /// \return true iff the given mask of shuffle node \p N is a XXPERMDI shuffle /// mask. bool PPC::isXXPERMDIShuffleMask(ShuffleVectorSDNode *N, unsigned &DM, bool &Swap, bool IsLE) { assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8"); // Ensure each byte index of the double word is consecutive. if (!isNByteElemShuffleMask(N, 8, 1)) return false; unsigned M0 = N->getMaskElt(0) / 8; unsigned M1 = N->getMaskElt(8) / 8; assert(((M0 | M1) < 4) && "A mask element out of bounds?"); // If both vector operands for the shuffle are the same vector, the mask will // contain only elements from the first one and the second one will be undef. if (N->getOperand(1).isUndef()) { if ((M0 | M1) < 2) { DM = IsLE ? (((~M1) & 1) << 1) + ((~M0) & 1) : (M0 << 1) + (M1 & 1); Swap = false; return true; } else return false; } if (IsLE) { if (M0 > 1 && M1 < 2) { Swap = false; } else if (M0 < 2 && M1 > 1) { M0 = (M0 + 2) % 4; M1 = (M1 + 2) % 4; Swap = true; } else return false; // Note: if control flow comes here that means Swap is already set above DM = (((~M1) & 1) << 1) + ((~M0) & 1); return true; } else { // BE if (M0 < 2 && M1 > 1) { Swap = false; } else if (M0 > 1 && M1 < 2) { M0 = (M0 + 2) % 4; M1 = (M1 + 2) % 4; Swap = true; } else return false; // Note: if control flow comes here that means Swap is already set above DM = (M0 << 1) + (M1 & 1); return true; } } /// getVSPLTImmediate - Return the appropriate VSPLT* immediate to splat the /// specified isSplatShuffleMask VECTOR_SHUFFLE mask. unsigned PPC::getVSPLTImmediate(SDNode *N, unsigned EltSize, SelectionDAG &DAG) { ShuffleVectorSDNode *SVOp = cast(N); assert(isSplatShuffleMask(SVOp, EltSize)); if (DAG.getDataLayout().isLittleEndian()) return (16 / EltSize) - 1 - (SVOp->getMaskElt(0) / EltSize); else return SVOp->getMaskElt(0) / EltSize; } /// get_VSPLTI_elt - If this is a build_vector of constants which can be formed /// by using a vspltis[bhw] instruction of the specified element size, return /// the constant being splatted. The ByteSize field indicates the number of /// bytes of each element [124] -> [bhw]. SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) { SDValue OpVal(nullptr, 0); // If ByteSize of the splat is bigger than the element size of the // build_vector, then we have a case where we are checking for a splat where // multiple elements of the buildvector are folded together into a single // logical element of the splat (e.g. "vsplish 1" to splat {0,1}*8). unsigned EltSize = 16/N->getNumOperands(); if (EltSize < ByteSize) { unsigned Multiple = ByteSize/EltSize; // Number of BV entries per spltval. SDValue UniquedVals[4]; assert(Multiple > 1 && Multiple <= 4 && "How can this happen?"); // See if all of the elements in the buildvector agree across. for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { if (N->getOperand(i).isUndef()) continue; // If the element isn't a constant, bail fully out. if (!isa(N->getOperand(i))) return SDValue(); if (!UniquedVals[i&(Multiple-1)].getNode()) UniquedVals[i&(Multiple-1)] = N->getOperand(i); else if (UniquedVals[i&(Multiple-1)] != N->getOperand(i)) return SDValue(); // no match. } // Okay, if we reached this point, UniquedVals[0..Multiple-1] contains // either constant or undef values that are identical for each chunk. See // if these chunks can form into a larger vspltis*. // Check to see if all of the leading entries are either 0 or -1. If // neither, then this won't fit into the immediate field. bool LeadingZero = true; bool LeadingOnes = true; for (unsigned i = 0; i != Multiple-1; ++i) { if (!UniquedVals[i].getNode()) continue; // Must have been undefs. LeadingZero &= isNullConstant(UniquedVals[i]); LeadingOnes &= isAllOnesConstant(UniquedVals[i]); } // Finally, check the least significant entry. if (LeadingZero) { if (!UniquedVals[Multiple-1].getNode()) return DAG.getTargetConstant(0, SDLoc(N), MVT::i32); // 0,0,0,undef int Val = cast(UniquedVals[Multiple-1])->getZExtValue(); if (Val < 16) // 0,0,0,4 -> vspltisw(4) return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32); } if (LeadingOnes) { if (!UniquedVals[Multiple-1].getNode()) return DAG.getTargetConstant(~0U, SDLoc(N), MVT::i32); // -1,-1,-1,undef int Val =cast(UniquedVals[Multiple-1])->getSExtValue(); if (Val >= -16) // -1,-1,-1,-2 -> vspltisw(-2) return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32); } return SDValue(); } // Check to see if this buildvec has a single non-undef value in its elements. for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { if (N->getOperand(i).isUndef()) continue; if (!OpVal.getNode()) OpVal = N->getOperand(i); else if (OpVal != N->getOperand(i)) return SDValue(); } if (!OpVal.getNode()) return SDValue(); // All UNDEF: use implicit def. unsigned ValSizeInBytes = EltSize; uint64_t Value = 0; if (ConstantSDNode *CN = dyn_cast(OpVal)) { Value = CN->getZExtValue(); } else if (ConstantFPSDNode *CN = dyn_cast(OpVal)) { assert(CN->getValueType(0) == MVT::f32 && "Only one legal FP vector type!"); Value = FloatToBits(CN->getValueAPF().convertToFloat()); } // If the splat value is larger than the element value, then we can never do // this splat. The only case that we could fit the replicated bits into our // immediate field for would be zero, and we prefer to use vxor for it. if (ValSizeInBytes < ByteSize) return SDValue(); // If the element value is larger than the splat value, check if it consists // of a repeated bit pattern of size ByteSize. if (!APInt(ValSizeInBytes * 8, Value).isSplat(ByteSize * 8)) return SDValue(); // Properly sign extend the value. int MaskVal = SignExtend32(Value, ByteSize * 8); // If this is zero, don't match, zero matches ISD::isBuildVectorAllZeros. if (MaskVal == 0) return SDValue(); // Finally, if this value fits in a 5 bit sext field, return it if (SignExtend32<5>(MaskVal) == MaskVal) return DAG.getTargetConstant(MaskVal, SDLoc(N), MVT::i32); return SDValue(); } /// isQVALIGNIShuffleMask - If this is a qvaligni shuffle mask, return the shift /// amount, otherwise return -1. int PPC::isQVALIGNIShuffleMask(SDNode *N) { EVT VT = N->getValueType(0); if (VT != MVT::v4f64 && VT != MVT::v4f32 && VT != MVT::v4i1) return -1; ShuffleVectorSDNode *SVOp = cast(N); // Find the first non-undef value in the shuffle mask. unsigned i; for (i = 0; i != 4 && SVOp->getMaskElt(i) < 0; ++i) /*search*/; if (i == 4) return -1; // all undef. // Otherwise, check to see if the rest of the elements are consecutively // numbered from this value. unsigned ShiftAmt = SVOp->getMaskElt(i); if (ShiftAmt < i) return -1; ShiftAmt -= i; // Check the rest of the elements to see if they are consecutive. for (++i; i != 4; ++i) if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i)) return -1; return ShiftAmt; } //===----------------------------------------------------------------------===// // Addressing Mode Selection //===----------------------------------------------------------------------===// /// isIntS16Immediate - This method tests to see if the node is either a 32-bit /// or 64-bit immediate, and if the value can be accurately represented as a /// sign extension from a 16-bit value. If so, this returns true and the /// immediate. bool llvm::isIntS16Immediate(SDNode *N, int16_t &Imm) { if (!isa(N)) return false; Imm = (int16_t)cast(N)->getZExtValue(); if (N->getValueType(0) == MVT::i32) return Imm == (int32_t)cast(N)->getZExtValue(); else return Imm == (int64_t)cast(N)->getZExtValue(); } bool llvm::isIntS16Immediate(SDValue Op, int16_t &Imm) { return isIntS16Immediate(Op.getNode(), Imm); } /// SelectAddressEVXRegReg - Given the specified address, check to see if it can /// be represented as an indexed [r+r] operation. bool PPCTargetLowering::SelectAddressEVXRegReg(SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG) const { for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E; ++UI) { if (MemSDNode *Memop = dyn_cast(*UI)) { if (Memop->getMemoryVT() == MVT::f64) { Base = N.getOperand(0); Index = N.getOperand(1); return true; } } } return false; } /// SelectAddressRegReg - Given the specified addressed, check to see if it /// can be represented as an indexed [r+r] operation. Returns false if it /// can be more efficiently represented as [r+imm]. If \p EncodingAlignment is /// non-zero and N can be represented by a base register plus a signed 16-bit /// displacement, make a more precise judgement by checking (displacement % \p /// EncodingAlignment). bool PPCTargetLowering::SelectAddressRegReg(SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG, unsigned EncodingAlignment) const { int16_t imm = 0; if (N.getOpcode() == ISD::ADD) { // Is there any SPE load/store (f64), which can't handle 16bit offset? // SPE load/store can only handle 8-bit offsets. if (hasSPE() && SelectAddressEVXRegReg(N, Base, Index, DAG)) return true; if (isIntS16Immediate(N.getOperand(1), imm) && (!EncodingAlignment || !(imm % EncodingAlignment))) return false; // r+i if (N.getOperand(1).getOpcode() == PPCISD::Lo) return false; // r+i Base = N.getOperand(0); Index = N.getOperand(1); return true; } else if (N.getOpcode() == ISD::OR) { if (isIntS16Immediate(N.getOperand(1), imm) && (!EncodingAlignment || !(imm % EncodingAlignment))) return false; // r+i can fold it if we can. // If this is an or of disjoint bitfields, we can codegen this as an add // (for better address arithmetic) if the LHS and RHS of the OR are provably // disjoint. KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0)); if (LHSKnown.Zero.getBoolValue()) { KnownBits RHSKnown = DAG.computeKnownBits(N.getOperand(1)); // If all of the bits are known zero on the LHS or RHS, the add won't // carry. if (~(LHSKnown.Zero | RHSKnown.Zero) == 0) { Base = N.getOperand(0); Index = N.getOperand(1); return true; } } } return false; } // If we happen to be doing an i64 load or store into a stack slot that has // less than a 4-byte alignment, then the frame-index elimination may need to // use an indexed load or store instruction (because the offset may not be a // multiple of 4). The extra register needed to hold the offset comes from the // register scavenger, and it is possible that the scavenger will need to use // an emergency spill slot. As a result, we need to make sure that a spill slot // is allocated when doing an i64 load/store into a less-than-4-byte-aligned // stack slot. static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) { // FIXME: This does not handle the LWA case. if (VT != MVT::i64) return; // NOTE: We'll exclude negative FIs here, which come from argument // lowering, because there are no known test cases triggering this problem // using packed structures (or similar). We can remove this exclusion if // we find such a test case. The reason why this is so test-case driven is // because this entire 'fixup' is only to prevent crashes (from the // register scavenger) on not-really-valid inputs. For example, if we have: // %a = alloca i1 // %b = bitcast i1* %a to i64* // store i64* a, i64 b // then the store should really be marked as 'align 1', but is not. If it // were marked as 'align 1' then the indexed form would have been // instruction-selected initially, and the problem this 'fixup' is preventing // won't happen regardless. if (FrameIdx < 0) return; MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); unsigned Align = MFI.getObjectAlignment(FrameIdx); if (Align >= 4) return; PPCFunctionInfo *FuncInfo = MF.getInfo(); FuncInfo->setHasNonRISpills(); } /// Returns true if the address N can be represented by a base register plus /// a signed 16-bit displacement [r+imm], and if it is not better /// represented as reg+reg. If \p EncodingAlignment is non-zero, only accept /// displacements that are multiples of that value. bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp, SDValue &Base, SelectionDAG &DAG, unsigned EncodingAlignment) const { // FIXME dl should come from parent load or store, not from address SDLoc dl(N); // If this can be more profitably realized as r+r, fail. if (SelectAddressRegReg(N, Disp, Base, DAG, EncodingAlignment)) return false; if (N.getOpcode() == ISD::ADD) { int16_t imm = 0; if (isIntS16Immediate(N.getOperand(1), imm) && (!EncodingAlignment || (imm % EncodingAlignment) == 0)) { Disp = DAG.getTargetConstant(imm, dl, N.getValueType()); if (FrameIndexSDNode *FI = dyn_cast(N.getOperand(0))) { Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); } else { Base = N.getOperand(0); } return true; // [r+i] } else if (N.getOperand(1).getOpcode() == PPCISD::Lo) { // Match LOAD (ADD (X, Lo(G))). assert(!cast(N.getOperand(1).getOperand(1))->getZExtValue() && "Cannot handle constant offsets yet!"); Disp = N.getOperand(1).getOperand(0); // The global address. assert(Disp.getOpcode() == ISD::TargetGlobalAddress || Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable); Base = N.getOperand(0); return true; // [&g+r] } } else if (N.getOpcode() == ISD::OR) { int16_t imm = 0; if (isIntS16Immediate(N.getOperand(1), imm) && (!EncodingAlignment || (imm % EncodingAlignment) == 0)) { // If this is an or of disjoint bitfields, we can codegen this as an add // (for better address arithmetic) if the LHS and RHS of the OR are // provably disjoint. KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0)); if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)imm) == ~0ULL) { // If all of the bits are known zero on the LHS or RHS, the add won't // carry. if (FrameIndexSDNode *FI = dyn_cast(N.getOperand(0))) { Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); } else { Base = N.getOperand(0); } Disp = DAG.getTargetConstant(imm, dl, N.getValueType()); return true; } } } else if (ConstantSDNode *CN = dyn_cast(N)) { // Loading from a constant address. // If this address fits entirely in a 16-bit sext immediate field, codegen // this as "d, 0" int16_t Imm; if (isIntS16Immediate(CN, Imm) && (!EncodingAlignment || (Imm % EncodingAlignment) == 0)) { Disp = DAG.getTargetConstant(Imm, dl, CN->getValueType(0)); Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, CN->getValueType(0)); return true; } // Handle 32-bit sext immediates with LIS + addr mode. if ((CN->getValueType(0) == MVT::i32 || (int64_t)CN->getZExtValue() == (int)CN->getZExtValue()) && (!EncodingAlignment || (CN->getZExtValue() % EncodingAlignment) == 0)) { int Addr = (int)CN->getZExtValue(); // Otherwise, break this down into an LIS + disp. Disp = DAG.getTargetConstant((short)Addr, dl, MVT::i32); Base = DAG.getTargetConstant((Addr - (signed short)Addr) >> 16, dl, MVT::i32); unsigned Opc = CN->getValueType(0) == MVT::i32 ? PPC::LIS : PPC::LIS8; Base = SDValue(DAG.getMachineNode(Opc, dl, CN->getValueType(0), Base), 0); return true; } } Disp = DAG.getTargetConstant(0, dl, getPointerTy(DAG.getDataLayout())); if (FrameIndexSDNode *FI = dyn_cast(N)) { Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); } else Base = N; return true; // [r+0] } /// SelectAddressRegRegOnly - Given the specified addressed, force it to be /// represented as an indexed [r+r] operation. bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG) const { // Check to see if we can easily represent this as an [r+r] address. This // will fail if it thinks that the address is more profitably represented as // reg+imm, e.g. where imm = 0. if (SelectAddressRegReg(N, Base, Index, DAG)) return true; // If the address is the result of an add, we will utilize the fact that the // address calculation includes an implicit add. However, we can reduce // register pressure if we do not materialize a constant just for use as the // index register. We only get rid of the add if it is not an add of a // value and a 16-bit signed constant and both have a single use. int16_t imm = 0; if (N.getOpcode() == ISD::ADD && (!isIntS16Immediate(N.getOperand(1), imm) || !N.getOperand(1).hasOneUse() || !N.getOperand(0).hasOneUse())) { Base = N.getOperand(0); Index = N.getOperand(1); return true; } // Otherwise, do it the hard way, using R0 as the base register. Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, N.getValueType()); Index = N; return true; } /// Returns true if we should use a direct load into vector instruction /// (such as lxsd or lfd), instead of a load into gpr + direct move sequence. static bool usePartialVectorLoads(SDNode *N, const PPCSubtarget& ST) { // If there are any other uses other than scalar to vector, then we should // keep it as a scalar load -> direct move pattern to prevent multiple // loads. LoadSDNode *LD = dyn_cast(N); if (!LD) return false; EVT MemVT = LD->getMemoryVT(); if (!MemVT.isSimple()) return false; switch(MemVT.getSimpleVT().SimpleTy) { case MVT::i64: break; case MVT::i32: if (!ST.hasP8Vector()) return false; break; case MVT::i16: case MVT::i8: if (!ST.hasP9Vector()) return false; break; default: return false; } SDValue LoadedVal(N, 0); if (!LoadedVal.hasOneUse()) return false; for (SDNode::use_iterator UI = LD->use_begin(), UE = LD->use_end(); UI != UE; ++UI) if (UI.getUse().get().getResNo() == 0 && UI->getOpcode() != ISD::SCALAR_TO_VECTOR) return false; return true; } /// getPreIndexedAddressParts - returns true by value, base pointer and /// offset pointer and addressing mode by reference if the node's address /// can be legally represented as pre-indexed load / store address. bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const { if (DisablePPCPreinc) return false; bool isLoad = true; SDValue Ptr; EVT VT; unsigned Alignment; if (LoadSDNode *LD = dyn_cast(N)) { Ptr = LD->getBasePtr(); VT = LD->getMemoryVT(); Alignment = LD->getAlignment(); } else if (StoreSDNode *ST = dyn_cast(N)) { Ptr = ST->getBasePtr(); VT = ST->getMemoryVT(); Alignment = ST->getAlignment(); isLoad = false; } else return false; // Do not generate pre-inc forms for specific loads that feed scalar_to_vector // instructions because we can fold these into a more efficient instruction // instead, (such as LXSD). if (isLoad && usePartialVectorLoads(N, Subtarget)) { return false; } // PowerPC doesn't have preinc load/store instructions for vectors (except // for QPX, which does have preinc r+r forms). if (VT.isVector()) { if (!Subtarget.hasQPX() || (VT != MVT::v4f64 && VT != MVT::v4f32)) { return false; } else if (SelectAddressRegRegOnly(Ptr, Offset, Base, DAG)) { AM = ISD::PRE_INC; return true; } } if (SelectAddressRegReg(Ptr, Base, Offset, DAG)) { // Common code will reject creating a pre-inc form if the base pointer // is a frame index, or if N is a store and the base pointer is either // the same as or a predecessor of the value being stored. Check for // those situations here, and try with swapped Base/Offset instead. bool Swap = false; if (isa(Base) || isa(Base)) Swap = true; else if (!isLoad) { SDValue Val = cast(N)->getValue(); if (Val == Base || Base.getNode()->isPredecessorOf(Val.getNode())) Swap = true; } if (Swap) std::swap(Base, Offset); AM = ISD::PRE_INC; return true; } // LDU/STU can only handle immediates that are a multiple of 4. if (VT != MVT::i64) { if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, 0)) return false; } else { // LDU/STU need an address with at least 4-byte alignment. if (Alignment < 4) return false; if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, 4)) return false; } if (LoadSDNode *LD = dyn_cast(N)) { // PPC64 doesn't have lwau, but it does have lwaux. Reject preinc load of // sext i32 to i64 when addr mode is r+i. if (LD->getValueType(0) == MVT::i64 && LD->getMemoryVT() == MVT::i32 && LD->getExtensionType() == ISD::SEXTLOAD && isa(Offset)) return false; } AM = ISD::PRE_INC; return true; } //===----------------------------------------------------------------------===// // LowerOperation implementation //===----------------------------------------------------------------------===// /// Return true if we should reference labels using a PICBase, set the HiOpFlags /// and LoOpFlags to the target MO flags. static void getLabelAccessInfo(bool IsPIC, const PPCSubtarget &Subtarget, unsigned &HiOpFlags, unsigned &LoOpFlags, const GlobalValue *GV = nullptr) { HiOpFlags = PPCII::MO_HA; LoOpFlags = PPCII::MO_LO; // Don't use the pic base if not in PIC relocation model. if (IsPIC) { HiOpFlags |= PPCII::MO_PIC_FLAG; LoOpFlags |= PPCII::MO_PIC_FLAG; } // If this is a reference to a global value that requires a non-lazy-ptr, make // sure that instruction lowering adds it. if (GV && Subtarget.hasLazyResolverStub(GV)) { HiOpFlags |= PPCII::MO_NLP_FLAG; LoOpFlags |= PPCII::MO_NLP_FLAG; if (GV->hasHiddenVisibility()) { HiOpFlags |= PPCII::MO_NLP_HIDDEN_FLAG; LoOpFlags |= PPCII::MO_NLP_HIDDEN_FLAG; } } } static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC, SelectionDAG &DAG) { SDLoc DL(HiPart); EVT PtrVT = HiPart.getValueType(); SDValue Zero = DAG.getConstant(0, DL, PtrVT); SDValue Hi = DAG.getNode(PPCISD::Hi, DL, PtrVT, HiPart, Zero); SDValue Lo = DAG.getNode(PPCISD::Lo, DL, PtrVT, LoPart, Zero); // With PIC, the first instruction is actually "GR+hi(&G)". if (isPIC) Hi = DAG.getNode(ISD::ADD, DL, PtrVT, DAG.getNode(PPCISD::GlobalBaseReg, DL, PtrVT), Hi); // Generate non-pic code that has direct accesses to the constant pool. // The address of the global is just (hi(&g)+lo(&g)). return DAG.getNode(ISD::ADD, DL, PtrVT, Hi, Lo); } static void setUsesTOCBasePtr(MachineFunction &MF) { PPCFunctionInfo *FuncInfo = MF.getInfo(); FuncInfo->setUsesTOCBasePtr(); } static void setUsesTOCBasePtr(SelectionDAG &DAG) { setUsesTOCBasePtr(DAG.getMachineFunction()); } static SDValue getTOCEntry(SelectionDAG &DAG, const SDLoc &dl, bool Is64Bit, SDValue GA) { EVT VT = Is64Bit ? MVT::i64 : MVT::i32; SDValue Reg = Is64Bit ? DAG.getRegister(PPC::X2, VT) : DAG.getNode(PPCISD::GlobalBaseReg, dl, VT); SDValue Ops[] = { GA, Reg }; return DAG.getMemIntrinsicNode( PPCISD::TOC_ENTRY, dl, DAG.getVTList(VT, MVT::Other), Ops, VT, MachinePointerInfo::getGOT(DAG.getMachineFunction()), 0, MachineMemOperand::MOLoad); } SDValue PPCTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const { EVT PtrVT = Op.getValueType(); ConstantPoolSDNode *CP = cast(Op); const Constant *C = CP->getConstVal(); // 64-bit SVR4 ABI code is always position-independent. // The actual address of the GlobalValue is stored in the TOC. if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) { setUsesTOCBasePtr(DAG); SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0); return getTOCEntry(DAG, SDLoc(CP), true, GA); } unsigned MOHiFlag, MOLoFlag; bool IsPIC = isPositionIndependent(); getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag); if (IsPIC && Subtarget.isSVR4ABI()) { SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), PPCII::MO_PIC_FLAG); return getTOCEntry(DAG, SDLoc(CP), false, GA); } SDValue CPIHi = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0, MOHiFlag); SDValue CPILo = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0, MOLoFlag); return LowerLabelRef(CPIHi, CPILo, IsPIC, DAG); } // For 64-bit PowerPC, prefer the more compact relative encodings. // This trades 32 bits per jump table entry for one or two instructions // on the jump site. unsigned PPCTargetLowering::getJumpTableEncoding() const { if (isJumpTableRelative()) return MachineJumpTableInfo::EK_LabelDifference32; return TargetLowering::getJumpTableEncoding(); } bool PPCTargetLowering::isJumpTableRelative() const { if (Subtarget.isPPC64()) return true; return TargetLowering::isJumpTableRelative(); } SDValue PPCTargetLowering::getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) const { if (!Subtarget.isPPC64()) return TargetLowering::getPICJumpTableRelocBase(Table, DAG); switch (getTargetMachine().getCodeModel()) { case CodeModel::Small: case CodeModel::Medium: return TargetLowering::getPICJumpTableRelocBase(Table, DAG); default: return DAG.getNode(PPCISD::GlobalBaseReg, SDLoc(), getPointerTy(DAG.getDataLayout())); } } const MCExpr * PPCTargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const { if (!Subtarget.isPPC64()) return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx); switch (getTargetMachine().getCodeModel()) { case CodeModel::Small: case CodeModel::Medium: return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx); default: return MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx); } } SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const { EVT PtrVT = Op.getValueType(); JumpTableSDNode *JT = cast(Op); // 64-bit SVR4 ABI code is always position-independent. // The actual address of the GlobalValue is stored in the TOC. if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) { setUsesTOCBasePtr(DAG); SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); return getTOCEntry(DAG, SDLoc(JT), true, GA); } unsigned MOHiFlag, MOLoFlag; bool IsPIC = isPositionIndependent(); getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag); if (IsPIC && Subtarget.isSVR4ABI()) { SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, PPCII::MO_PIC_FLAG); return getTOCEntry(DAG, SDLoc(GA), false, GA); } SDValue JTIHi = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOHiFlag); SDValue JTILo = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOLoFlag); return LowerLabelRef(JTIHi, JTILo, IsPIC, DAG); } SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const { EVT PtrVT = Op.getValueType(); BlockAddressSDNode *BASDN = cast(Op); const BlockAddress *BA = BASDN->getBlockAddress(); // 64-bit SVR4 ABI code is always position-independent. // The actual BlockAddress is stored in the TOC. if (Subtarget.isSVR4ABI() && (Subtarget.isPPC64() || isPositionIndependent())) { if (Subtarget.isPPC64()) setUsesTOCBasePtr(DAG); SDValue GA = DAG.getTargetBlockAddress(BA, PtrVT, BASDN->getOffset()); return getTOCEntry(DAG, SDLoc(BASDN), Subtarget.isPPC64(), GA); } unsigned MOHiFlag, MOLoFlag; bool IsPIC = isPositionIndependent(); getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag); SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOHiFlag); SDValue TgtBALo = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOLoFlag); return LowerLabelRef(TgtBAHi, TgtBALo, IsPIC, DAG); } SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { // FIXME: TLS addresses currently use medium model code sequences, // which is the most useful form. Eventually support for small and // large models could be added if users need it, at the cost of // additional complexity. GlobalAddressSDNode *GA = cast(Op); if (DAG.getTarget().useEmulatedTLS()) return LowerToTLSEmulatedModel(GA, DAG); SDLoc dl(GA); const GlobalValue *GV = GA->getGlobal(); EVT PtrVT = getPointerTy(DAG.getDataLayout()); bool is64bit = Subtarget.isPPC64(); const Module *M = DAG.getMachineFunction().getFunction().getParent(); PICLevel::Level picLevel = M->getPICLevel(); const TargetMachine &TM = getTargetMachine(); TLSModel::Model Model = TM.getTLSModel(GV); if (Model == TLSModel::LocalExec) { SDValue TGAHi = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TPREL_HA); SDValue TGALo = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TPREL_LO); SDValue TLSReg = is64bit ? DAG.getRegister(PPC::X13, MVT::i64) : DAG.getRegister(PPC::R2, MVT::i32); SDValue Hi = DAG.getNode(PPCISD::Hi, dl, PtrVT, TGAHi, TLSReg); return DAG.getNode(PPCISD::Lo, dl, PtrVT, TGALo, Hi); } if (Model == TLSModel::InitialExec) { SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); SDValue TGATLS = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TLS); SDValue GOTPtr; if (is64bit) { setUsesTOCBasePtr(DAG); SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); GOTPtr = DAG.getNode(PPCISD::ADDIS_GOT_TPREL_HA, dl, PtrVT, GOTReg, TGA); } else { if (!TM.isPositionIndependent()) GOTPtr = DAG.getNode(PPCISD::PPC32_GOT, dl, PtrVT); else if (picLevel == PICLevel::SmallPIC) GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); else GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); } SDValue TPOffset = DAG.getNode(PPCISD::LD_GOT_TPREL_L, dl, PtrVT, TGA, GOTPtr); return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TPOffset, TGATLS); } if (Model == TLSModel::GeneralDynamic) { SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); SDValue GOTPtr; if (is64bit) { setUsesTOCBasePtr(DAG); SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSGD_HA, dl, PtrVT, GOTReg, TGA); } else { if (picLevel == PICLevel::SmallPIC) GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); else GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); } return DAG.getNode(PPCISD::ADDI_TLSGD_L_ADDR, dl, PtrVT, GOTPtr, TGA, TGA); } if (Model == TLSModel::LocalDynamic) { SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); SDValue GOTPtr; if (is64bit) { setUsesTOCBasePtr(DAG); SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSLD_HA, dl, PtrVT, GOTReg, TGA); } else { if (picLevel == PICLevel::SmallPIC) GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); else GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); } SDValue TLSAddr = DAG.getNode(PPCISD::ADDI_TLSLD_L_ADDR, dl, PtrVT, GOTPtr, TGA, TGA); SDValue DtvOffsetHi = DAG.getNode(PPCISD::ADDIS_DTPREL_HA, dl, PtrVT, TLSAddr, TGA); return DAG.getNode(PPCISD::ADDI_DTPREL_L, dl, PtrVT, DtvOffsetHi, TGA); } llvm_unreachable("Unknown TLS model!"); } SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const { EVT PtrVT = Op.getValueType(); GlobalAddressSDNode *GSDN = cast(Op); SDLoc DL(GSDN); const GlobalValue *GV = GSDN->getGlobal(); // 64-bit SVR4 ABI code is always position-independent. // The actual address of the GlobalValue is stored in the TOC. if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) { setUsesTOCBasePtr(DAG); SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset()); return getTOCEntry(DAG, DL, true, GA); } unsigned MOHiFlag, MOLoFlag; bool IsPIC = isPositionIndependent(); getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag, GV); if (IsPIC && Subtarget.isSVR4ABI()) { SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), PPCII::MO_PIC_FLAG); return getTOCEntry(DAG, DL, false, GA); } SDValue GAHi = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOHiFlag); SDValue GALo = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOLoFlag); SDValue Ptr = LowerLabelRef(GAHi, GALo, IsPIC, DAG); // If the global reference is actually to a non-lazy-pointer, we have to do an // extra load to get the address of the global. if (MOHiFlag & PPCII::MO_NLP_FLAG) Ptr = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo()); return Ptr; } SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { ISD::CondCode CC = cast(Op.getOperand(2))->get(); SDLoc dl(Op); if (Op.getValueType() == MVT::v2i64) { // When the operands themselves are v2i64 values, we need to do something // special because VSX has no underlying comparison operations for these. if (Op.getOperand(0).getValueType() == MVT::v2i64) { // Equality can be handled by casting to the legal type for Altivec // comparisons, everything else needs to be expanded. if (CC == ISD::SETEQ || CC == ISD::SETNE) { return DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, DAG.getSetCC(dl, MVT::v4i32, DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(0)), DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(1)), CC)); } return SDValue(); } // We handle most of these in the usual way. return Op; } // If we're comparing for equality to zero, expose the fact that this is // implemented as a ctlz/srl pair on ppc, so that the dag combiner can // fold the new nodes. if (SDValue V = lowerCmpEqZeroToCtlzSrl(Op, DAG)) return V; if (ConstantSDNode *C = dyn_cast(Op.getOperand(1))) { // Leave comparisons against 0 and -1 alone for now, since they're usually // optimized. FIXME: revisit this when we can custom lower all setcc // optimizations. if (C->isAllOnesValue() || C->isNullValue()) return SDValue(); } // If we have an integer seteq/setne, turn it into a compare against zero // by xor'ing the rhs with the lhs, which is faster than setting a // condition register, reading it back out, and masking the correct bit. The // normal approach here uses sub to do this instead of xor. Using xor exposes // the result to other bit-twiddling opportunities. EVT LHSVT = Op.getOperand(0).getValueType(); if (LHSVT.isInteger() && (CC == ISD::SETEQ || CC == ISD::SETNE)) { EVT VT = Op.getValueType(); SDValue Sub = DAG.getNode(ISD::XOR, dl, LHSVT, Op.getOperand(0), Op.getOperand(1)); return DAG.getSetCC(dl, VT, Sub, DAG.getConstant(0, dl, LHSVT), CC); } return SDValue(); } SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const { SDNode *Node = Op.getNode(); EVT VT = Node->getValueType(0); EVT PtrVT = getPointerTy(DAG.getDataLayout()); SDValue InChain = Node->getOperand(0); SDValue VAListPtr = Node->getOperand(1); const Value *SV = cast(Node->getOperand(2))->getValue(); SDLoc dl(Node); assert(!Subtarget.isPPC64() && "LowerVAARG is PPC32 only"); // gpr_index SDValue GprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain, VAListPtr, MachinePointerInfo(SV), MVT::i8); InChain = GprIndex.getValue(1); if (VT == MVT::i64) { // Check if GprIndex is even SDValue GprAnd = DAG.getNode(ISD::AND, dl, MVT::i32, GprIndex, DAG.getConstant(1, dl, MVT::i32)); SDValue CC64 = DAG.getSetCC(dl, MVT::i32, GprAnd, DAG.getConstant(0, dl, MVT::i32), ISD::SETNE); SDValue GprIndexPlusOne = DAG.getNode(ISD::ADD, dl, MVT::i32, GprIndex, DAG.getConstant(1, dl, MVT::i32)); // Align GprIndex to be even if it isn't GprIndex = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC64, GprIndexPlusOne, GprIndex); } // fpr index is 1 byte after gpr SDValue FprPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, DAG.getConstant(1, dl, MVT::i32)); // fpr SDValue FprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain, FprPtr, MachinePointerInfo(SV), MVT::i8); InChain = FprIndex.getValue(1); SDValue RegSaveAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, DAG.getConstant(8, dl, MVT::i32)); SDValue OverflowAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, DAG.getConstant(4, dl, MVT::i32)); // areas SDValue OverflowArea = DAG.getLoad(MVT::i32, dl, InChain, OverflowAreaPtr, MachinePointerInfo()); InChain = OverflowArea.getValue(1); SDValue RegSaveArea = DAG.getLoad(MVT::i32, dl, InChain, RegSaveAreaPtr, MachinePointerInfo()); InChain = RegSaveArea.getValue(1); // select overflow_area if index > 8 SDValue CC = DAG.getSetCC(dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex, DAG.getConstant(8, dl, MVT::i32), ISD::SETLT); // adjustment constant gpr_index * 4/8 SDValue RegConstant = DAG.getNode(ISD::MUL, dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex, DAG.getConstant(VT.isInteger() ? 4 : 8, dl, MVT::i32)); // OurReg = RegSaveArea + RegConstant SDValue OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, RegSaveArea, RegConstant); // Floating types are 32 bytes into RegSaveArea if (VT.isFloatingPoint()) OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, OurReg, DAG.getConstant(32, dl, MVT::i32)); // increase {f,g}pr_index by 1 (or 2 if VT is i64) SDValue IndexPlus1 = DAG.getNode(ISD::ADD, dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex, DAG.getConstant(VT == MVT::i64 ? 2 : 1, dl, MVT::i32)); InChain = DAG.getTruncStore(InChain, dl, IndexPlus1, VT.isInteger() ? VAListPtr : FprPtr, MachinePointerInfo(SV), MVT::i8); // determine if we should load from reg_save_area or overflow_area SDValue Result = DAG.getNode(ISD::SELECT, dl, PtrVT, CC, OurReg, OverflowArea); // increase overflow_area by 4/8 if gpr/fpr > 8 SDValue OverflowAreaPlusN = DAG.getNode(ISD::ADD, dl, PtrVT, OverflowArea, DAG.getConstant(VT.isInteger() ? 4 : 8, dl, MVT::i32)); OverflowArea = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC, OverflowArea, OverflowAreaPlusN); InChain = DAG.getTruncStore(InChain, dl, OverflowArea, OverflowAreaPtr, MachinePointerInfo(), MVT::i32); return DAG.getLoad(VT, dl, InChain, Result, MachinePointerInfo()); } SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const { assert(!Subtarget.isPPC64() && "LowerVACOPY is PPC32 only"); // We have to copy the entire va_list struct: // 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte return DAG.getMemcpy(Op.getOperand(0), Op, Op.getOperand(1), Op.getOperand(2), DAG.getConstant(12, SDLoc(Op), MVT::i32), 8, false, true, false, MachinePointerInfo(), MachinePointerInfo()); } SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const { return Op.getOperand(0); } SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const { SDValue Chain = Op.getOperand(0); SDValue Trmp = Op.getOperand(1); // trampoline SDValue FPtr = Op.getOperand(2); // nested function SDValue Nest = Op.getOperand(3); // 'nest' parameter value SDLoc dl(Op); EVT PtrVT = getPointerTy(DAG.getDataLayout()); bool isPPC64 = (PtrVT == MVT::i64); Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext()); TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; Entry.Ty = IntPtrTy; Entry.Node = Trmp; Args.push_back(Entry); // TrampSize == (isPPC64 ? 48 : 40); Entry.Node = DAG.getConstant(isPPC64 ? 48 : 40, dl, isPPC64 ? MVT::i64 : MVT::i32); Args.push_back(Entry); Entry.Node = FPtr; Args.push_back(Entry); Entry.Node = Nest; Args.push_back(Entry); // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg) TargetLowering::CallLoweringInfo CLI(DAG); CLI.setDebugLoc(dl).setChain(Chain).setLibCallee( CallingConv::C, Type::getVoidTy(*DAG.getContext()), DAG.getExternalSymbol("__trampoline_setup", PtrVT), std::move(Args)); std::pair CallResult = LowerCallTo(CLI); return CallResult.second; } SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const { MachineFunction &MF = DAG.getMachineFunction(); PPCFunctionInfo *FuncInfo = MF.getInfo(); EVT PtrVT = getPointerTy(MF.getDataLayout()); SDLoc dl(Op); if (Subtarget.isDarwinABI() || Subtarget.isPPC64()) { // vastart just stores the address of the VarArgsFrameIndex slot into the // memory location argument. SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); const Value *SV = cast(Op.getOperand(2))->getValue(); return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1), MachinePointerInfo(SV)); } // For the 32-bit SVR4 ABI we follow the layout of the va_list struct. // We suppose the given va_list is already allocated. // // typedef struct { // char gpr; /* index into the array of 8 GPRs // * stored in the register save area // * gpr=0 corresponds to r3, // * gpr=1 to r4, etc. // */ // char fpr; /* index into the array of 8 FPRs // * stored in the register save area // * fpr=0 corresponds to f1, // * fpr=1 to f2, etc. // */ // char *overflow_arg_area; // /* location on stack that holds // * the next overflow argument // */ // char *reg_save_area; // /* where r3:r10 and f1:f8 (if saved) // * are stored // */ // } va_list[1]; SDValue ArgGPR = DAG.getConstant(FuncInfo->getVarArgsNumGPR(), dl, MVT::i32); SDValue ArgFPR = DAG.getConstant(FuncInfo->getVarArgsNumFPR(), dl, MVT::i32); SDValue StackOffsetFI = DAG.getFrameIndex(FuncInfo->getVarArgsStackOffset(), PtrVT); SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); uint64_t FrameOffset = PtrVT.getSizeInBits()/8; SDValue ConstFrameOffset = DAG.getConstant(FrameOffset, dl, PtrVT); uint64_t StackOffset = PtrVT.getSizeInBits()/8 - 1; SDValue ConstStackOffset = DAG.getConstant(StackOffset, dl, PtrVT); uint64_t FPROffset = 1; SDValue ConstFPROffset = DAG.getConstant(FPROffset, dl, PtrVT); const Value *SV = cast(Op.getOperand(2))->getValue(); // Store first byte : number of int regs SDValue firstStore = DAG.getTruncStore(Op.getOperand(0), dl, ArgGPR, Op.getOperand(1), MachinePointerInfo(SV), MVT::i8); uint64_t nextOffset = FPROffset; SDValue nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, Op.getOperand(1), ConstFPROffset); // Store second byte : number of float regs SDValue secondStore = DAG.getTruncStore(firstStore, dl, ArgFPR, nextPtr, MachinePointerInfo(SV, nextOffset), MVT::i8); nextOffset += StackOffset; nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstStackOffset); // Store second word : arguments given on stack SDValue thirdStore = DAG.getStore(secondStore, dl, StackOffsetFI, nextPtr, MachinePointerInfo(SV, nextOffset)); nextOffset += FrameOffset; nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstFrameOffset); // Store third word : arguments given in registers return DAG.getStore(thirdStore, dl, FR, nextPtr, MachinePointerInfo(SV, nextOffset)); } /// FPR - The set of FP registers that should be allocated for arguments, /// on Darwin. static const MCPhysReg FPR[] = {PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7, PPC::F8, PPC::F9, PPC::F10, PPC::F11, PPC::F12, PPC::F13}; /// QFPR - The set of QPX registers that should be allocated for arguments. static const MCPhysReg QFPR[] = { PPC::QF1, PPC::QF2, PPC::QF3, PPC::QF4, PPC::QF5, PPC::QF6, PPC::QF7, PPC::QF8, PPC::QF9, PPC::QF10, PPC::QF11, PPC::QF12, PPC::QF13}; /// CalculateStackSlotSize - Calculates the size reserved for this argument on /// the stack. static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags, unsigned PtrByteSize) { unsigned ArgSize = ArgVT.getStoreSize(); if (Flags.isByVal()) ArgSize = Flags.getByValSize(); // Round up to multiples of the pointer size, except for array members, // which are always packed. if (!Flags.isInConsecutiveRegs()) ArgSize = ((ArgSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; return ArgSize; } /// CalculateStackSlotAlignment - Calculates the alignment of this argument /// on the stack. static unsigned CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT, ISD::ArgFlagsTy Flags, unsigned PtrByteSize) { unsigned Align = PtrByteSize; // Altivec parameters are padded to a 16 byte boundary. if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 || ArgVT == MVT::v1i128 || ArgVT == MVT::f128) Align = 16; // QPX vector types stored in double-precision are padded to a 32 byte // boundary. else if (ArgVT == MVT::v4f64 || ArgVT == MVT::v4i1) Align = 32; // ByVal parameters are aligned as requested. if (Flags.isByVal()) { unsigned BVAlign = Flags.getByValAlign(); if (BVAlign > PtrByteSize) { if (BVAlign % PtrByteSize != 0) llvm_unreachable( "ByVal alignment is not a multiple of the pointer size"); Align = BVAlign; } } // Array members are always packed to their original alignment. if (Flags.isInConsecutiveRegs()) { // If the array member was split into multiple registers, the first // needs to be aligned to the size of the full type. (Except for // ppcf128, which is only aligned as its f64 components.) if (Flags.isSplit() && OrigVT != MVT::ppcf128) Align = OrigVT.getStoreSize(); else Align = ArgVT.getStoreSize(); } return Align; } /// CalculateStackSlotUsed - Return whether this argument will use its /// stack slot (instead of being passed in registers). ArgOffset, /// AvailableFPRs, and AvailableVRs must hold the current argument /// position, and will be updated to account for this argument. static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, ISD::ArgFlagsTy Flags, unsigned PtrByteSize, unsigned LinkageSize, unsigned ParamAreaSize, unsigned &ArgOffset, unsigned &AvailableFPRs, unsigned &AvailableVRs, bool HasQPX) { bool UseMemory = false; // Respect alignment of argument on the stack. unsigned Align = CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); ArgOffset = ((ArgOffset + Align - 1) / Align) * Align; // If there's no space left in the argument save area, we must // use memory (this check also catches zero-sized arguments). if (ArgOffset >= LinkageSize + ParamAreaSize) UseMemory = true; // Allocate argument on the stack. ArgOffset += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); if (Flags.isInConsecutiveRegsLast()) ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; // If we overran the argument save area, we must use memory // (this check catches arguments passed partially in memory) if (ArgOffset > LinkageSize + ParamAreaSize) UseMemory = true; // However, if the argument is actually passed in an FPR or a VR, // we don't use memory after all. if (!Flags.isByVal()) { if (ArgVT == MVT::f32 || ArgVT == MVT::f64 || // QPX registers overlap with the scalar FP registers. (HasQPX && (ArgVT == MVT::v4f32 || ArgVT == MVT::v4f64 || ArgVT == MVT::v4i1))) if (AvailableFPRs > 0) { --AvailableFPRs; return false; } if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 || ArgVT == MVT::v1i128 || ArgVT == MVT::f128) if (AvailableVRs > 0) { --AvailableVRs; return false; } } return UseMemory; } /// EnsureStackAlignment - Round stack frame size up from NumBytes to /// ensure minimum alignment required for target. static unsigned EnsureStackAlignment(const PPCFrameLowering *Lowering, unsigned NumBytes) { unsigned TargetAlign = Lowering->getStackAlignment(); unsigned AlignMask = TargetAlign - 1; NumBytes = (NumBytes + AlignMask) & ~AlignMask; return NumBytes; } SDValue PPCTargetLowering::LowerFormalArguments( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const { if (Subtarget.isSVR4ABI()) { if (Subtarget.isPPC64()) return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG, InVals); else return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG, InVals); } else { return LowerFormalArguments_Darwin(Chain, CallConv, isVarArg, Ins, dl, DAG, InVals); } } SDValue PPCTargetLowering::LowerFormalArguments_32SVR4( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const { // 32-bit SVR4 ABI Stack Frame Layout: // +-----------------------------------+ // +--> | Back chain | // | +-----------------------------------+ // | | Floating-point register save area | // | +-----------------------------------+ // | | General register save area | // | +-----------------------------------+ // | | CR save word | // | +-----------------------------------+ // | | VRSAVE save word | // | +-----------------------------------+ // | | Alignment padding | // | +-----------------------------------+ // | | Vector register save area | // | +-----------------------------------+ // | | Local variable space | // | +-----------------------------------+ // | | Parameter list area | // | +-----------------------------------+ // | | LR save word | // | +-----------------------------------+ // SP--> +--- | Back chain | // +-----------------------------------+ // // Specifications: // System V Application Binary Interface PowerPC Processor Supplement // AltiVec Technology Programming Interface Manual MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); PPCFunctionInfo *FuncInfo = MF.getInfo(); EVT PtrVT = getPointerTy(MF.getDataLayout()); // Potential tail calls could cause overwriting of argument stack slots. bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && (CallConv == CallingConv::Fast)); unsigned PtrByteSize = 4; // Assign locations to all of the incoming arguments. SmallVector ArgLocs; PPCCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, *DAG.getContext()); // Reserve space for the linkage area on the stack. unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); CCInfo.AllocateStack(LinkageSize, PtrByteSize); if (useSoftFloat()) CCInfo.PreAnalyzeFormalArguments(Ins); CCInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4); CCInfo.clearWasPPCF128(); for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { CCValAssign &VA = ArgLocs[i]; // Arguments stored in registers. if (VA.isRegLoc()) { const TargetRegisterClass *RC; EVT ValVT = VA.getValVT(); switch (ValVT.getSimpleVT().SimpleTy) { default: llvm_unreachable("ValVT not supported by formal arguments Lowering"); case MVT::i1: case MVT::i32: RC = &PPC::GPRCRegClass; break; case MVT::f32: if (Subtarget.hasP8Vector()) RC = &PPC::VSSRCRegClass; else if (Subtarget.hasSPE()) RC = &PPC::SPE4RCRegClass; else RC = &PPC::F4RCRegClass; break; case MVT::f64: if (Subtarget.hasVSX()) RC = &PPC::VSFRCRegClass; else if (Subtarget.hasSPE()) // SPE passes doubles in GPR pairs. RC = &PPC::GPRCRegClass; else RC = &PPC::F8RCRegClass; break; case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: RC = &PPC::VRRCRegClass; break; case MVT::v4f32: RC = Subtarget.hasQPX() ? &PPC::QSRCRegClass : &PPC::VRRCRegClass; break; case MVT::v2f64: case MVT::v2i64: RC = &PPC::VRRCRegClass; break; case MVT::v4f64: RC = &PPC::QFRCRegClass; break; case MVT::v4i1: RC = &PPC::QBRCRegClass; break; } SDValue ArgValue; // Transform the arguments stored in physical registers into // virtual ones. if (VA.getLocVT() == MVT::f64 && Subtarget.hasSPE()) { assert(i + 1 < e && "No second half of double precision argument"); unsigned RegLo = MF.addLiveIn(VA.getLocReg(), RC); unsigned RegHi = MF.addLiveIn(ArgLocs[++i].getLocReg(), RC); SDValue ArgValueLo = DAG.getCopyFromReg(Chain, dl, RegLo, MVT::i32); SDValue ArgValueHi = DAG.getCopyFromReg(Chain, dl, RegHi, MVT::i32); if (!Subtarget.isLittleEndian()) std::swap (ArgValueLo, ArgValueHi); ArgValue = DAG.getNode(PPCISD::BUILD_SPE64, dl, MVT::f64, ArgValueLo, ArgValueHi); } else { unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC); ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, ValVT == MVT::i1 ? MVT::i32 : ValVT); if (ValVT == MVT::i1) ArgValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgValue); } InVals.push_back(ArgValue); } else { // Argument stored in memory. assert(VA.isMemLoc()); // Get the extended size of the argument type in stack unsigned ArgSize = VA.getLocVT().getStoreSize(); // Get the actual size of the argument type unsigned ObjSize = VA.getValVT().getStoreSize(); unsigned ArgOffset = VA.getLocMemOffset(); // Stack objects in PPC32 are right justified. ArgOffset += ArgSize - ObjSize; int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, isImmutable); // Create load nodes to retrieve arguments from the stack. SDValue FIN = DAG.getFrameIndex(FI, PtrVT); InVals.push_back( DAG.getLoad(VA.getValVT(), dl, Chain, FIN, MachinePointerInfo())); } } // Assign locations to all of the incoming aggregate by value arguments. // Aggregates passed by value are stored in the local variable space of the // caller's stack frame, right above the parameter list area. SmallVector ByValArgLocs; CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(), ByValArgLocs, *DAG.getContext()); // Reserve stack space for the allocations in CCInfo. CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrByteSize); CCByValInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4_ByVal); // Area that is at least reserved in the caller of this function. unsigned MinReservedArea = CCByValInfo.getNextStackOffset(); MinReservedArea = std::max(MinReservedArea, LinkageSize); // Set the size that is at least reserved in caller of this function. Tail // call optimized function's reserved stack space needs to be aligned so that // taking the difference between two stack areas will result in an aligned // stack. MinReservedArea = EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); FuncInfo->setMinReservedArea(MinReservedArea); SmallVector MemOps; // If the function takes variable number of arguments, make a frame index for // the start of the first vararg value... for expansion of llvm.va_start. if (isVarArg) { static const MCPhysReg GPArgRegs[] = { PPC::R3, PPC::R4, PPC::R5, PPC::R6, PPC::R7, PPC::R8, PPC::R9, PPC::R10, }; const unsigned NumGPArgRegs = array_lengthof(GPArgRegs); static const MCPhysReg FPArgRegs[] = { PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7, PPC::F8 }; unsigned NumFPArgRegs = array_lengthof(FPArgRegs); if (useSoftFloat() || hasSPE()) NumFPArgRegs = 0; FuncInfo->setVarArgsNumGPR(CCInfo.getFirstUnallocated(GPArgRegs)); FuncInfo->setVarArgsNumFPR(CCInfo.getFirstUnallocated(FPArgRegs)); // Make room for NumGPArgRegs and NumFPArgRegs. int Depth = NumGPArgRegs * PtrVT.getSizeInBits()/8 + NumFPArgRegs * MVT(MVT::f64).getSizeInBits()/8; FuncInfo->setVarArgsStackOffset( MFI.CreateFixedObject(PtrVT.getSizeInBits()/8, CCInfo.getNextStackOffset(), true)); FuncInfo->setVarArgsFrameIndex(MFI.CreateStackObject(Depth, 8, false)); SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); // The fixed integer arguments of a variadic function are stored to the // VarArgsFrameIndex on the stack so that they may be loaded by // dereferencing the result of va_next. for (unsigned GPRIndex = 0; GPRIndex != NumGPArgRegs; ++GPRIndex) { // Get an existing live-in vreg, or add a new one. unsigned VReg = MF.getRegInfo().getLiveInVirtReg(GPArgRegs[GPRIndex]); if (!VReg) VReg = MF.addLiveIn(GPArgRegs[GPRIndex], &PPC::GPRCRegClass); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); MemOps.push_back(Store); // Increment the address by four for the next argument to store SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT); FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); } // FIXME 32-bit SVR4: We only need to save FP argument registers if CR bit 6 // is set. // The double arguments are stored to the VarArgsFrameIndex // on the stack. for (unsigned FPRIndex = 0; FPRIndex != NumFPArgRegs; ++FPRIndex) { // Get an existing live-in vreg, or add a new one. unsigned VReg = MF.getRegInfo().getLiveInVirtReg(FPArgRegs[FPRIndex]); if (!VReg) VReg = MF.addLiveIn(FPArgRegs[FPRIndex], &PPC::F8RCRegClass); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::f64); SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); MemOps.push_back(Store); // Increment the address by eight for the next argument to store SDValue PtrOff = DAG.getConstant(MVT(MVT::f64).getSizeInBits()/8, dl, PtrVT); FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); } } if (!MemOps.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); return Chain; } // PPC64 passes i8, i16, and i32 values in i64 registers. Promote // value to MVT::i64 and then truncate to the correct register size. SDValue PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags, EVT ObjectVT, SelectionDAG &DAG, SDValue ArgVal, const SDLoc &dl) const { if (Flags.isSExt()) ArgVal = DAG.getNode(ISD::AssertSext, dl, MVT::i64, ArgVal, DAG.getValueType(ObjectVT)); else if (Flags.isZExt()) ArgVal = DAG.getNode(ISD::AssertZext, dl, MVT::i64, ArgVal, DAG.getValueType(ObjectVT)); return DAG.getNode(ISD::TRUNCATE, dl, ObjectVT, ArgVal); } SDValue PPCTargetLowering::LowerFormalArguments_64SVR4( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const { // TODO: add description of PPC stack frame format, or at least some docs. // bool isELFv2ABI = Subtarget.isELFv2ABI(); bool isLittleEndian = Subtarget.isLittleEndian(); MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); PPCFunctionInfo *FuncInfo = MF.getInfo(); assert(!(CallConv == CallingConv::Fast && isVarArg) && "fastcc not supported on varargs functions"); EVT PtrVT = getPointerTy(MF.getDataLayout()); // Potential tail calls could cause overwriting of argument stack slots. bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && (CallConv == CallingConv::Fast)); unsigned PtrByteSize = 8; unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); static const MCPhysReg GPR[] = { PPC::X3, PPC::X4, PPC::X5, PPC::X6, PPC::X7, PPC::X8, PPC::X9, PPC::X10, }; static const MCPhysReg VR[] = { PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 }; const unsigned Num_GPR_Regs = array_lengthof(GPR); const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13; const unsigned Num_VR_Regs = array_lengthof(VR); const unsigned Num_QFPR_Regs = Num_FPR_Regs; // Do a first pass over the arguments to determine whether the ABI // guarantees that our caller has allocated the parameter save area // on its stack frame. In the ELFv1 ABI, this is always the case; // in the ELFv2 ABI, it is true if this is a vararg function or if // any parameter is located in a stack slot. bool HasParameterArea = !isELFv2ABI || isVarArg; unsigned ParamAreaSize = Num_GPR_Regs * PtrByteSize; unsigned NumBytes = LinkageSize; unsigned AvailableFPRs = Num_FPR_Regs; unsigned AvailableVRs = Num_VR_Regs; for (unsigned i = 0, e = Ins.size(); i != e; ++i) { if (Ins[i].Flags.isNest()) continue; if (CalculateStackSlotUsed(Ins[i].VT, Ins[i].ArgVT, Ins[i].Flags, PtrByteSize, LinkageSize, ParamAreaSize, NumBytes, AvailableFPRs, AvailableVRs, Subtarget.hasQPX())) HasParameterArea = true; } // Add DAG nodes to load the arguments or copy them out of registers. On // entry to a function on PPC, the arguments start after the linkage area, // although the first ones are often in registers. unsigned ArgOffset = LinkageSize; unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; unsigned &QFPR_idx = FPR_idx; SmallVector MemOps; Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin(); unsigned CurArgIdx = 0; for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { SDValue ArgVal; bool needsLoad = false; EVT ObjectVT = Ins[ArgNo].VT; EVT OrigVT = Ins[ArgNo].ArgVT; unsigned ObjSize = ObjectVT.getStoreSize(); unsigned ArgSize = ObjSize; ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; if (Ins[ArgNo].isOrigArg()) { std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx); CurArgIdx = Ins[ArgNo].getOrigArgIndex(); } // We re-align the argument offset for each argument, except when using the // fast calling convention, when we need to make sure we do that only when // we'll actually use a stack slot. unsigned CurArgOffset, Align; auto ComputeArgOffset = [&]() { /* Respect alignment of argument on the stack. */ Align = CalculateStackSlotAlignment(ObjectVT, OrigVT, Flags, PtrByteSize); ArgOffset = ((ArgOffset + Align - 1) / Align) * Align; CurArgOffset = ArgOffset; }; if (CallConv != CallingConv::Fast) { ComputeArgOffset(); /* Compute GPR index associated with argument offset. */ GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; GPR_idx = std::min(GPR_idx, Num_GPR_Regs); } // FIXME the codegen can be much improved in some cases. // We do not have to keep everything in memory. if (Flags.isByVal()) { assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit"); if (CallConv == CallingConv::Fast) ComputeArgOffset(); // ObjSize is the true size, ArgSize rounded up to multiple of registers. ObjSize = Flags.getByValSize(); ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; // Empty aggregate parameters do not take up registers. Examples: // struct { } a; // union { } b; // int c[0]; // etc. However, we have to provide a place-holder in InVals, so // pretend we have an 8-byte item at the current address for that // purpose. if (!ObjSize) { int FI = MFI.CreateFixedObject(PtrByteSize, ArgOffset, true); SDValue FIN = DAG.getFrameIndex(FI, PtrVT); InVals.push_back(FIN); continue; } // Create a stack object covering all stack doublewords occupied // by the argument. If the argument is (fully or partially) on // the stack, or if the argument is fully in registers but the // caller has allocated the parameter save anyway, we can refer // directly to the caller's stack frame. Otherwise, create a // local copy in our own frame. int FI; if (HasParameterArea || ArgSize + ArgOffset > LinkageSize + Num_GPR_Regs * PtrByteSize) FI = MFI.CreateFixedObject(ArgSize, ArgOffset, false, true); else FI = MFI.CreateStackObject(ArgSize, Align, false); SDValue FIN = DAG.getFrameIndex(FI, PtrVT); // Handle aggregates smaller than 8 bytes. if (ObjSize < PtrByteSize) { // The value of the object is its address, which differs from the // address of the enclosing doubleword on big-endian systems. SDValue Arg = FIN; if (!isLittleEndian) { SDValue ArgOff = DAG.getConstant(PtrByteSize - ObjSize, dl, PtrVT); Arg = DAG.getNode(ISD::ADD, dl, ArgOff.getValueType(), Arg, ArgOff); } InVals.push_back(Arg); if (GPR_idx != Num_GPR_Regs) { unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); FuncInfo->addLiveInAttr(VReg, Flags); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); SDValue Store; if (ObjSize==1 || ObjSize==2 || ObjSize==4) { EVT ObjType = (ObjSize == 1 ? MVT::i8 : (ObjSize == 2 ? MVT::i16 : MVT::i32)); Store = DAG.getTruncStore(Val.getValue(1), dl, Val, Arg, MachinePointerInfo(&*FuncArg), ObjType); } else { // For sizes that don't fit a truncating store (3, 5, 6, 7), // store the whole register as-is to the parameter save area // slot. Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo(&*FuncArg)); } MemOps.push_back(Store); } // Whether we copied from a register or not, advance the offset // into the parameter save area by a full doubleword. ArgOffset += PtrByteSize; continue; } // The value of the object is its address, which is the address of // its first stack doubleword. InVals.push_back(FIN); // Store whatever pieces of the object are in registers to memory. for (unsigned j = 0; j < ArgSize; j += PtrByteSize) { if (GPR_idx == Num_GPR_Regs) break; unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); FuncInfo->addLiveInAttr(VReg, Flags); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); SDValue Addr = FIN; if (j) { SDValue Off = DAG.getConstant(j, dl, PtrVT); Addr = DAG.getNode(ISD::ADD, dl, Off.getValueType(), Addr, Off); } SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, Addr, MachinePointerInfo(&*FuncArg, j)); MemOps.push_back(Store); ++GPR_idx; } ArgOffset += ArgSize; continue; } switch (ObjectVT.getSimpleVT().SimpleTy) { default: llvm_unreachable("Unhandled argument type!"); case MVT::i1: case MVT::i32: case MVT::i64: if (Flags.isNest()) { // The 'nest' parameter, if any, is passed in R11. unsigned VReg = MF.addLiveIn(PPC::X11, &PPC::G8RCRegClass); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); break; } // These can be scalar arguments or elements of an integer array type // passed directly. Clang may use those instead of "byval" aggregate // types to avoid forcing arguments to memory unnecessarily. if (GPR_idx != Num_GPR_Regs) { unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); FuncInfo->addLiveInAttr(VReg, Flags); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) // PPC64 passes i8, i16, and i32 values in i64 registers. Promote // value to MVT::i64 and then truncate to the correct register size. ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); } else { if (CallConv == CallingConv::Fast) ComputeArgOffset(); needsLoad = true; ArgSize = PtrByteSize; } if (CallConv != CallingConv::Fast || needsLoad) ArgOffset += 8; break; case MVT::f32: case MVT::f64: // These can be scalar arguments or elements of a float array type // passed directly. The latter are used to implement ELFv2 homogenous // float aggregates. if (FPR_idx != Num_FPR_Regs) { unsigned VReg; if (ObjectVT == MVT::f32) VReg = MF.addLiveIn(FPR[FPR_idx], Subtarget.hasP8Vector() ? &PPC::VSSRCRegClass : &PPC::F4RCRegClass); else VReg = MF.addLiveIn(FPR[FPR_idx], Subtarget.hasVSX() ? &PPC::VSFRCRegClass : &PPC::F8RCRegClass); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); ++FPR_idx; } else if (GPR_idx != Num_GPR_Regs && CallConv != CallingConv::Fast) { // FIXME: We may want to re-enable this for CallingConv::Fast on the P8 // once we support fp <-> gpr moves. // This can only ever happen in the presence of f32 array types, // since otherwise we never run out of FPRs before running out // of GPRs. unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); FuncInfo->addLiveInAttr(VReg, Flags); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); if (ObjectVT == MVT::f32) { if ((ArgOffset % PtrByteSize) == (isLittleEndian ? 4 : 0)) ArgVal = DAG.getNode(ISD::SRL, dl, MVT::i64, ArgVal, DAG.getConstant(32, dl, MVT::i32)); ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, ArgVal); } ArgVal = DAG.getNode(ISD::BITCAST, dl, ObjectVT, ArgVal); } else { if (CallConv == CallingConv::Fast) ComputeArgOffset(); needsLoad = true; } // When passing an array of floats, the array occupies consecutive // space in the argument area; only round up to the next doubleword // at the end of the array. Otherwise, each float takes 8 bytes. if (CallConv != CallingConv::Fast || needsLoad) { ArgSize = Flags.isInConsecutiveRegs() ? ObjSize : PtrByteSize; ArgOffset += ArgSize; if (Flags.isInConsecutiveRegsLast()) ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; } break; case MVT::v4f32: case MVT::v4i32: case MVT::v8i16: case MVT::v16i8: case MVT::v2f64: case MVT::v2i64: case MVT::v1i128: case MVT::f128: if (!Subtarget.hasQPX()) { // These can be scalar arguments or elements of a vector array type // passed directly. The latter are used to implement ELFv2 homogenous // vector aggregates. if (VR_idx != Num_VR_Regs) { unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); ++VR_idx; } else { if (CallConv == CallingConv::Fast) ComputeArgOffset(); needsLoad = true; } if (CallConv != CallingConv::Fast || needsLoad) ArgOffset += 16; break; } // not QPX assert(ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 && "Invalid QPX parameter type"); LLVM_FALLTHROUGH; case MVT::v4f64: case MVT::v4i1: // QPX vectors are treated like their scalar floating-point subregisters // (except that they're larger). unsigned Sz = ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 ? 16 : 32; if (QFPR_idx != Num_QFPR_Regs) { const TargetRegisterClass *RC; switch (ObjectVT.getSimpleVT().SimpleTy) { case MVT::v4f64: RC = &PPC::QFRCRegClass; break; case MVT::v4f32: RC = &PPC::QSRCRegClass; break; default: RC = &PPC::QBRCRegClass; break; } unsigned VReg = MF.addLiveIn(QFPR[QFPR_idx], RC); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); ++QFPR_idx; } else { if (CallConv == CallingConv::Fast) ComputeArgOffset(); needsLoad = true; } if (CallConv != CallingConv::Fast || needsLoad) ArgOffset += Sz; break; } // We need to load the argument to a virtual register if we determined // above that we ran out of physical registers of the appropriate type. if (needsLoad) { if (ObjSize < ArgSize && !isLittleEndian) CurArgOffset += ArgSize - ObjSize; int FI = MFI.CreateFixedObject(ObjSize, CurArgOffset, isImmutable); SDValue FIN = DAG.getFrameIndex(FI, PtrVT); ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo()); } InVals.push_back(ArgVal); } // Area that is at least reserved in the caller of this function. unsigned MinReservedArea; if (HasParameterArea) MinReservedArea = std::max(ArgOffset, LinkageSize + 8 * PtrByteSize); else MinReservedArea = LinkageSize; // Set the size that is at least reserved in caller of this function. Tail // call optimized functions' reserved stack space needs to be aligned so that // taking the difference between two stack areas will result in an aligned // stack. MinReservedArea = EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); FuncInfo->setMinReservedArea(MinReservedArea); // If the function takes variable number of arguments, make a frame index for // the start of the first vararg value... for expansion of llvm.va_start. if (isVarArg) { int Depth = ArgOffset; FuncInfo->setVarArgsFrameIndex( MFI.CreateFixedObject(PtrByteSize, Depth, true)); SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); // If this function is vararg, store any remaining integer argument regs // to their spots on the stack so that they may be loaded by dereferencing // the result of va_next. for (GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; GPR_idx < Num_GPR_Regs; ++GPR_idx) { unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); MemOps.push_back(Store); // Increment the address by four for the next argument to store SDValue PtrOff = DAG.getConstant(PtrByteSize, dl, PtrVT); FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); } } if (!MemOps.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); return Chain; } SDValue PPCTargetLowering::LowerFormalArguments_Darwin( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const { // TODO: add description of PPC stack frame format, or at least some docs. // MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); PPCFunctionInfo *FuncInfo = MF.getInfo(); EVT PtrVT = getPointerTy(MF.getDataLayout()); bool isPPC64 = PtrVT == MVT::i64; // Potential tail calls could cause overwriting of argument stack slots. bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && (CallConv == CallingConv::Fast)); unsigned PtrByteSize = isPPC64 ? 8 : 4; unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); unsigned ArgOffset = LinkageSize; // Area that is at least reserved in caller of this function. unsigned MinReservedArea = ArgOffset; static const MCPhysReg GPR_32[] = { // 32-bit registers. PPC::R3, PPC::R4, PPC::R5, PPC::R6, PPC::R7, PPC::R8, PPC::R9, PPC::R10, }; static const MCPhysReg GPR_64[] = { // 64-bit registers. PPC::X3, PPC::X4, PPC::X5, PPC::X6, PPC::X7, PPC::X8, PPC::X9, PPC::X10, }; static const MCPhysReg VR[] = { PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 }; const unsigned Num_GPR_Regs = array_lengthof(GPR_32); const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13; const unsigned Num_VR_Regs = array_lengthof( VR); unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; // In 32-bit non-varargs functions, the stack space for vectors is after the // stack space for non-vectors. We do not use this space unless we have // too many vectors to fit in registers, something that only occurs in // constructed examples:), but we have to walk the arglist to figure // that out...for the pathological case, compute VecArgOffset as the // start of the vector parameter area. Computing VecArgOffset is the // entire point of the following loop. unsigned VecArgOffset = ArgOffset; if (!isVarArg && !isPPC64) { for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { EVT ObjectVT = Ins[ArgNo].VT; ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; if (Flags.isByVal()) { // ObjSize is the true size, ArgSize rounded up to multiple of regs. unsigned ObjSize = Flags.getByValSize(); unsigned ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; VecArgOffset += ArgSize; continue; } switch(ObjectVT.getSimpleVT().SimpleTy) { default: llvm_unreachable("Unhandled argument type!"); case MVT::i1: case MVT::i32: case MVT::f32: VecArgOffset += 4; break; case MVT::i64: // PPC64 case MVT::f64: // FIXME: We are guaranteed to be !isPPC64 at this point. // Does MVT::i64 apply? VecArgOffset += 8; break; case MVT::v4f32: case MVT::v4i32: case MVT::v8i16: case MVT::v16i8: // Nothing to do, we're only looking at Nonvector args here. break; } } } // We've found where the vector parameter area in memory is. Skip the // first 12 parameters; these don't use that memory. VecArgOffset = ((VecArgOffset+15)/16)*16; VecArgOffset += 12*16; // Add DAG nodes to load the arguments or copy them out of registers. On // entry to a function on PPC, the arguments start after the linkage area, // although the first ones are often in registers. SmallVector MemOps; unsigned nAltivecParamsAtEnd = 0; Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin(); unsigned CurArgIdx = 0; for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { SDValue ArgVal; bool needsLoad = false; EVT ObjectVT = Ins[ArgNo].VT; unsigned ObjSize = ObjectVT.getSizeInBits()/8; unsigned ArgSize = ObjSize; ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; if (Ins[ArgNo].isOrigArg()) { std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx); CurArgIdx = Ins[ArgNo].getOrigArgIndex(); } unsigned CurArgOffset = ArgOffset; // Varargs or 64 bit Altivec parameters are padded to a 16 byte boundary. if (ObjectVT==MVT::v4f32 || ObjectVT==MVT::v4i32 || ObjectVT==MVT::v8i16 || ObjectVT==MVT::v16i8) { if (isVarArg || isPPC64) { MinReservedArea = ((MinReservedArea+15)/16)*16; MinReservedArea += CalculateStackSlotSize(ObjectVT, Flags, PtrByteSize); } else nAltivecParamsAtEnd++; } else // Calculate min reserved area. MinReservedArea += CalculateStackSlotSize(Ins[ArgNo].VT, Flags, PtrByteSize); // FIXME the codegen can be much improved in some cases. // We do not have to keep everything in memory. if (Flags.isByVal()) { assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit"); // ObjSize is the true size, ArgSize rounded up to multiple of registers. ObjSize = Flags.getByValSize(); ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; // Objects of size 1 and 2 are right justified, everything else is // left justified. This means the memory address is adjusted forwards. if (ObjSize==1 || ObjSize==2) { CurArgOffset = CurArgOffset + (4 - ObjSize); } // The value of the object is its address. int FI = MFI.CreateFixedObject(ObjSize, CurArgOffset, false, true); SDValue FIN = DAG.getFrameIndex(FI, PtrVT); InVals.push_back(FIN); if (ObjSize==1 || ObjSize==2) { if (GPR_idx != Num_GPR_Regs) { unsigned VReg; if (isPPC64) VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); else VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); EVT ObjType = ObjSize == 1 ? MVT::i8 : MVT::i16; SDValue Store = DAG.getTruncStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo(&*FuncArg), ObjType); MemOps.push_back(Store); ++GPR_idx; } ArgOffset += PtrByteSize; continue; } for (unsigned j = 0; j < ArgSize; j += PtrByteSize) { // Store whatever pieces of the object are in registers // to memory. ArgOffset will be the address of the beginning // of the object. if (GPR_idx != Num_GPR_Regs) { unsigned VReg; if (isPPC64) VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); else VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); int FI = MFI.CreateFixedObject(PtrByteSize, ArgOffset, true); SDValue FIN = DAG.getFrameIndex(FI, PtrVT); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo(&*FuncArg, j)); MemOps.push_back(Store); ++GPR_idx; ArgOffset += PtrByteSize; } else { ArgOffset += ArgSize - (ArgOffset-CurArgOffset); break; } } continue; } switch (ObjectVT.getSimpleVT().SimpleTy) { default: llvm_unreachable("Unhandled argument type!"); case MVT::i1: case MVT::i32: if (!isPPC64) { if (GPR_idx != Num_GPR_Regs) { unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32); if (ObjectVT == MVT::i1) ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgVal); ++GPR_idx; } else { needsLoad = true; ArgSize = PtrByteSize; } // All int arguments reserve stack space in the Darwin ABI. ArgOffset += PtrByteSize; break; } LLVM_FALLTHROUGH; case MVT::i64: // PPC64 if (GPR_idx != Num_GPR_Regs) { unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) // PPC64 passes i8, i16, and i32 values in i64 registers. Promote // value to MVT::i64 and then truncate to the correct register size. ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); ++GPR_idx; } else { needsLoad = true; ArgSize = PtrByteSize; } // All int arguments reserve stack space in the Darwin ABI. ArgOffset += 8; break; case MVT::f32: case MVT::f64: // Every 4 bytes of argument space consumes one of the GPRs available for // argument passing. if (GPR_idx != Num_GPR_Regs) { ++GPR_idx; if (ObjSize == 8 && GPR_idx != Num_GPR_Regs && !isPPC64) ++GPR_idx; } if (FPR_idx != Num_FPR_Regs) { unsigned VReg; if (ObjectVT == MVT::f32) VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F4RCRegClass); else VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F8RCRegClass); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); ++FPR_idx; } else { needsLoad = true; } // All FP arguments reserve stack space in the Darwin ABI. ArgOffset += isPPC64 ? 8 : ObjSize; break; case MVT::v4f32: case MVT::v4i32: case MVT::v8i16: case MVT::v16i8: // Note that vector arguments in registers don't reserve stack space, // except in varargs functions. if (VR_idx != Num_VR_Regs) { unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); if (isVarArg) { while ((ArgOffset % 16) != 0) { ArgOffset += PtrByteSize; if (GPR_idx != Num_GPR_Regs) GPR_idx++; } ArgOffset += 16; GPR_idx = std::min(GPR_idx+4, Num_GPR_Regs); // FIXME correct for ppc64? } ++VR_idx; } else { if (!isVarArg && !isPPC64) { // Vectors go after all the nonvectors. CurArgOffset = VecArgOffset; VecArgOffset += 16; } else { // Vectors are aligned. ArgOffset = ((ArgOffset+15)/16)*16; CurArgOffset = ArgOffset; ArgOffset += 16; } needsLoad = true; } break; } // We need to load the argument to a virtual register if we determined above // that we ran out of physical registers of the appropriate type. if (needsLoad) { int FI = MFI.CreateFixedObject(ObjSize, CurArgOffset + (ArgSize - ObjSize), isImmutable); SDValue FIN = DAG.getFrameIndex(FI, PtrVT); ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo()); } InVals.push_back(ArgVal); } // Allow for Altivec parameters at the end, if needed. if (nAltivecParamsAtEnd) { MinReservedArea = ((MinReservedArea+15)/16)*16; MinReservedArea += 16*nAltivecParamsAtEnd; } // Area that is at least reserved in the caller of this function. MinReservedArea = std::max(MinReservedArea, LinkageSize + 8 * PtrByteSize); // Set the size that is at least reserved in caller of this function. Tail // call optimized functions' reserved stack space needs to be aligned so that // taking the difference between two stack areas will result in an aligned // stack. MinReservedArea = EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); FuncInfo->setMinReservedArea(MinReservedArea); // If the function takes variable number of arguments, make a frame index for // the start of the first vararg value... for expansion of llvm.va_start. if (isVarArg) { int Depth = ArgOffset; FuncInfo->setVarArgsFrameIndex( MFI.CreateFixedObject(PtrVT.getSizeInBits()/8, Depth, true)); SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); // If this function is vararg, store any remaining integer argument regs // to their spots on the stack so that they may be loaded by dereferencing // the result of va_next. for (; GPR_idx != Num_GPR_Regs; ++GPR_idx) { unsigned VReg; if (isPPC64) VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); else VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); MemOps.push_back(Store); // Increment the address by four for the next argument to store SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT); FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); } } if (!MemOps.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); return Chain; } /// CalculateTailCallSPDiff - Get the amount the stack pointer has to be /// adjusted to accommodate the arguments for the tailcall. static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall, unsigned ParamSize) { if (!isTailCall) return 0; PPCFunctionInfo *FI = DAG.getMachineFunction().getInfo(); unsigned CallerMinReservedArea = FI->getMinReservedArea(); int SPDiff = (int)CallerMinReservedArea - (int)ParamSize; // Remember only if the new adjustment is bigger. if (SPDiff < FI->getTailCallSPDelta()) FI->setTailCallSPDelta(SPDiff); return SPDiff; } static bool isFunctionGlobalAddress(SDValue Callee); static bool callsShareTOCBase(const Function *Caller, SDValue Callee, const TargetMachine &TM) { // Callee is either a GlobalAddress or an ExternalSymbol. ExternalSymbols // don't have enough information to determine if the caller and calle share // the same TOC base, so we have to pessimistically assume they don't for // correctness. GlobalAddressSDNode *G = dyn_cast(Callee); if (!G) return false; const GlobalValue *GV = G->getGlobal(); // The medium and large code models are expected to provide a sufficiently // large TOC to provide all data addressing needs of a module with a // single TOC. Since each module will be addressed with a single TOC then we // only need to check that caller and callee don't cross dso boundaries. if (CodeModel::Medium == TM.getCodeModel() || CodeModel::Large == TM.getCodeModel()) return TM.shouldAssumeDSOLocal(*Caller->getParent(), GV); // Otherwise we need to ensure callee and caller are in the same section, // since the linker may allocate multiple TOCs, and we don't know which // sections will belong to the same TOC base. if (!GV->isStrongDefinitionForLinker()) return false; // Any explicitly-specified sections and section prefixes must also match. // Also, if we're using -ffunction-sections, then each function is always in // a different section (the same is true for COMDAT functions). if (TM.getFunctionSections() || GV->hasComdat() || Caller->hasComdat() || GV->getSection() != Caller->getSection()) return false; if (const auto *F = dyn_cast(GV)) { if (F->getSectionPrefix() != Caller->getSectionPrefix()) return false; } // If the callee might be interposed, then we can't assume the ultimate call // target will be in the same section. Even in cases where we can assume that // interposition won't happen, in any case where the linker might insert a // stub to allow for interposition, we must generate code as though // interposition might occur. To understand why this matters, consider a // situation where: a -> b -> c where the arrows indicate calls. b and c are // in the same section, but a is in a different module (i.e. has a different // TOC base pointer). If the linker allows for interposition between b and c, // then it will generate a stub for the call edge between b and c which will // save the TOC pointer into the designated stack slot allocated by b. If we // return true here, and therefore allow a tail call between b and c, that // stack slot won't exist and the b -> c stub will end up saving b'c TOC base // pointer into the stack slot allocated by a (where the a -> b stub saved // a's TOC base pointer). If we're not considering a tail call, but rather, // whether a nop is needed after the call instruction in b, because the linker // will insert a stub, it might complain about a missing nop if we omit it // (although many don't complain in this case). if (!TM.shouldAssumeDSOLocal(*Caller->getParent(), GV)) return false; return true; } static bool needStackSlotPassParameters(const PPCSubtarget &Subtarget, const SmallVectorImpl &Outs) { assert(Subtarget.isSVR4ABI() && Subtarget.isPPC64()); const unsigned PtrByteSize = 8; const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); static const MCPhysReg GPR[] = { PPC::X3, PPC::X4, PPC::X5, PPC::X6, PPC::X7, PPC::X8, PPC::X9, PPC::X10, }; static const MCPhysReg VR[] = { PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 }; const unsigned NumGPRs = array_lengthof(GPR); const unsigned NumFPRs = 13; const unsigned NumVRs = array_lengthof(VR); const unsigned ParamAreaSize = NumGPRs * PtrByteSize; unsigned NumBytes = LinkageSize; unsigned AvailableFPRs = NumFPRs; unsigned AvailableVRs = NumVRs; for (const ISD::OutputArg& Param : Outs) { if (Param.Flags.isNest()) continue; if (CalculateStackSlotUsed(Param.VT, Param.ArgVT, Param.Flags, PtrByteSize, LinkageSize, ParamAreaSize, NumBytes, AvailableFPRs, AvailableVRs, Subtarget.hasQPX())) return true; } return false; } static bool hasSameArgumentList(const Function *CallerFn, ImmutableCallSite CS) { if (CS.arg_size() != CallerFn->arg_size()) return false; ImmutableCallSite::arg_iterator CalleeArgIter = CS.arg_begin(); ImmutableCallSite::arg_iterator CalleeArgEnd = CS.arg_end(); Function::const_arg_iterator CallerArgIter = CallerFn->arg_begin(); for (; CalleeArgIter != CalleeArgEnd; ++CalleeArgIter, ++CallerArgIter) { const Value* CalleeArg = *CalleeArgIter; const Value* CallerArg = &(*CallerArgIter); if (CalleeArg == CallerArg) continue; // e.g. @caller([4 x i64] %a, [4 x i64] %b) { // tail call @callee([4 x i64] undef, [4 x i64] %b) // } // 1st argument of callee is undef and has the same type as caller. if (CalleeArg->getType() == CallerArg->getType() && isa(CalleeArg)) continue; return false; } return true; } // Returns true if TCO is possible between the callers and callees // calling conventions. static bool areCallingConvEligibleForTCO_64SVR4(CallingConv::ID CallerCC, CallingConv::ID CalleeCC) { // Tail calls are possible with fastcc and ccc. auto isTailCallableCC = [] (CallingConv::ID CC){ return CC == CallingConv::C || CC == CallingConv::Fast; }; if (!isTailCallableCC(CallerCC) || !isTailCallableCC(CalleeCC)) return false; // We can safely tail call both fastcc and ccc callees from a c calling // convention caller. If the caller is fastcc, we may have less stack space // than a non-fastcc caller with the same signature so disable tail-calls in // that case. return CallerCC == CallingConv::C || CallerCC == CalleeCC; } bool PPCTargetLowering::IsEligibleForTailCallOptimization_64SVR4( SDValue Callee, CallingConv::ID CalleeCC, ImmutableCallSite CS, bool isVarArg, const SmallVectorImpl &Outs, const SmallVectorImpl &Ins, SelectionDAG& DAG) const { bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt; if (DisableSCO && !TailCallOpt) return false; // Variadic argument functions are not supported. if (isVarArg) return false; auto &Caller = DAG.getMachineFunction().getFunction(); // Check that the calling conventions are compatible for tco. if (!areCallingConvEligibleForTCO_64SVR4(Caller.getCallingConv(), CalleeCC)) return false; // Caller contains any byval parameter is not supported. if (any_of(Ins, [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); })) return false; // Callee contains any byval parameter is not supported, too. // Note: This is a quick work around, because in some cases, e.g. // caller's stack size > callee's stack size, we are still able to apply // sibling call optimization. For example, gcc is able to do SCO for caller1 // in the following example, but not for caller2. // struct test { // long int a; // char ary[56]; // } gTest; // __attribute__((noinline)) int callee(struct test v, struct test *b) { // b->a = v.a; // return 0; // } // void caller1(struct test a, struct test c, struct test *b) { // callee(gTest, b); } // void caller2(struct test *b) { callee(gTest, b); } if (any_of(Outs, [](const ISD::OutputArg& OA) { return OA.Flags.isByVal(); })) return false; // If callee and caller use different calling conventions, we cannot pass // parameters on stack since offsets for the parameter area may be different. if (Caller.getCallingConv() != CalleeCC && needStackSlotPassParameters(Subtarget, Outs)) return false; // No TCO/SCO on indirect call because Caller have to restore its TOC if (!isFunctionGlobalAddress(Callee) && !isa(Callee)) return false; // If the caller and callee potentially have different TOC bases then we // cannot tail call since we need to restore the TOC pointer after the call. // ref: https://bugzilla.mozilla.org/show_bug.cgi?id=973977 if (!callsShareTOCBase(&Caller, Callee, getTargetMachine())) return false; // TCO allows altering callee ABI, so we don't have to check further. if (CalleeCC == CallingConv::Fast && TailCallOpt) return true; if (DisableSCO) return false; // If callee use the same argument list that caller is using, then we can // apply SCO on this case. If it is not, then we need to check if callee needs // stack for passing arguments. if (!hasSameArgumentList(&Caller, CS) && needStackSlotPassParameters(Subtarget, Outs)) { return false; } return true; } /// IsEligibleForTailCallOptimization - Check whether the call is eligible /// for tail call optimization. Targets which want to do tail call /// optimization should implement this function. bool PPCTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg, const SmallVectorImpl &Ins, SelectionDAG& DAG) const { if (!getTargetMachine().Options.GuaranteedTailCallOpt) return false; // Variable argument functions are not supported. if (isVarArg) return false; MachineFunction &MF = DAG.getMachineFunction(); CallingConv::ID CallerCC = MF.getFunction().getCallingConv(); if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) { // Functions containing by val parameters are not supported. for (unsigned i = 0; i != Ins.size(); i++) { ISD::ArgFlagsTy Flags = Ins[i].Flags; if (Flags.isByVal()) return false; } // Non-PIC/GOT tail calls are supported. if (getTargetMachine().getRelocationModel() != Reloc::PIC_) return true; // At the moment we can only do local tail calls (in same module, hidden // or protected) if we are generating PIC. if (GlobalAddressSDNode *G = dyn_cast(Callee)) return G->getGlobal()->hasHiddenVisibility() || G->getGlobal()->hasProtectedVisibility(); } return false; } /// isCallCompatibleAddress - Return the immediate to use if the specified /// 32-bit value is representable in the immediate field of a BxA instruction. static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) { ConstantSDNode *C = dyn_cast(Op); if (!C) return nullptr; int Addr = C->getZExtValue(); if ((Addr & 3) != 0 || // Low 2 bits are implicitly zero. SignExtend32<26>(Addr) != Addr) return nullptr; // Top 6 bits have to be sext of immediate. return DAG .getConstant( (int)C->getZExtValue() >> 2, SDLoc(Op), DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout())) .getNode(); } namespace { struct TailCallArgumentInfo { SDValue Arg; SDValue FrameIdxOp; int FrameIdx = 0; TailCallArgumentInfo() = default; }; } // end anonymous namespace /// StoreTailCallArgumentsToStackSlot - Stores arguments to their stack slot. static void StoreTailCallArgumentsToStackSlot( SelectionDAG &DAG, SDValue Chain, const SmallVectorImpl &TailCallArgs, SmallVectorImpl &MemOpChains, const SDLoc &dl) { for (unsigned i = 0, e = TailCallArgs.size(); i != e; ++i) { SDValue Arg = TailCallArgs[i].Arg; SDValue FIN = TailCallArgs[i].FrameIdxOp; int FI = TailCallArgs[i].FrameIdx; // Store relative to framepointer. MemOpChains.push_back(DAG.getStore( Chain, dl, Arg, FIN, MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI))); } } /// EmitTailCallStoreFPAndRetAddr - Move the frame pointer and return address to /// the appropriate stack slot for the tail call optimized function call. static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG, SDValue Chain, SDValue OldRetAddr, SDValue OldFP, int SPDiff, const SDLoc &dl) { if (SPDiff) { // Calculate the new stack slot for the return address. MachineFunction &MF = DAG.getMachineFunction(); const PPCSubtarget &Subtarget = MF.getSubtarget(); const PPCFrameLowering *FL = Subtarget.getFrameLowering(); bool isPPC64 = Subtarget.isPPC64(); int SlotSize = isPPC64 ? 8 : 4; int NewRetAddrLoc = SPDiff + FL->getReturnSaveOffset(); int NewRetAddr = MF.getFrameInfo().CreateFixedObject(SlotSize, NewRetAddrLoc, true); EVT VT = isPPC64 ? MVT::i64 : MVT::i32; SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewRetAddr, VT); Chain = DAG.getStore(Chain, dl, OldRetAddr, NewRetAddrFrIdx, MachinePointerInfo::getFixedStack(MF, NewRetAddr)); // When using the 32/64-bit SVR4 ABI there is no need to move the FP stack // slot as the FP is never overwritten. if (Subtarget.isDarwinABI()) { int NewFPLoc = SPDiff + FL->getFramePointerSaveOffset(); int NewFPIdx = MF.getFrameInfo().CreateFixedObject(SlotSize, NewFPLoc, true); SDValue NewFramePtrIdx = DAG.getFrameIndex(NewFPIdx, VT); Chain = DAG.getStore(Chain, dl, OldFP, NewFramePtrIdx, MachinePointerInfo::getFixedStack( DAG.getMachineFunction(), NewFPIdx)); } } return Chain; } /// CalculateTailCallArgDest - Remember Argument for later processing. Calculate /// the position of the argument. static void CalculateTailCallArgDest(SelectionDAG &DAG, MachineFunction &MF, bool isPPC64, SDValue Arg, int SPDiff, unsigned ArgOffset, SmallVectorImpl& TailCallArguments) { int Offset = ArgOffset + SPDiff; uint32_t OpSize = (Arg.getValueSizeInBits() + 7) / 8; int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true); EVT VT = isPPC64 ? MVT::i64 : MVT::i32; SDValue FIN = DAG.getFrameIndex(FI, VT); TailCallArgumentInfo Info; Info.Arg = Arg; Info.FrameIdxOp = FIN; Info.FrameIdx = FI; TailCallArguments.push_back(Info); } /// EmitTCFPAndRetAddrLoad - Emit load from frame pointer and return address /// stack slot. Returns the chain as result and the loaded frame pointers in /// LROpOut/FPOpout. Used when tail calling. SDValue PPCTargetLowering::EmitTailCallLoadFPAndRetAddr( SelectionDAG &DAG, int SPDiff, SDValue Chain, SDValue &LROpOut, SDValue &FPOpOut, const SDLoc &dl) const { if (SPDiff) { // Load the LR and FP stack slot for later adjusting. EVT VT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32; LROpOut = getReturnAddrFrameIndex(DAG); LROpOut = DAG.getLoad(VT, dl, Chain, LROpOut, MachinePointerInfo()); Chain = SDValue(LROpOut.getNode(), 1); // When using the 32/64-bit SVR4 ABI there is no need to load the FP stack // slot as the FP is never overwritten. if (Subtarget.isDarwinABI()) { FPOpOut = getFramePointerFrameIndex(DAG); FPOpOut = DAG.getLoad(VT, dl, Chain, FPOpOut, MachinePointerInfo()); Chain = SDValue(FPOpOut.getNode(), 1); } } return Chain; } /// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified /// by "Src" to address "Dst" of size "Size". Alignment information is /// specified by the specific parameter attribute. The copy will be passed as /// a byval function parameter. /// Sometimes what we are copying is the end of a larger object, the part that /// does not fit in registers. static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, const SDLoc &dl) { SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32); return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(), false, false, false, MachinePointerInfo(), MachinePointerInfo()); } /// LowerMemOpCallTo - Store the argument to the stack or remember it in case of /// tail calls. static void LowerMemOpCallTo( SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue Arg, SDValue PtrOff, int SPDiff, unsigned ArgOffset, bool isPPC64, bool isTailCall, bool isVector, SmallVectorImpl &MemOpChains, SmallVectorImpl &TailCallArguments, const SDLoc &dl) { EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); if (!isTailCall) { if (isVector) { SDValue StackPtr; if (isPPC64) StackPtr = DAG.getRegister(PPC::X1, MVT::i64); else StackPtr = DAG.getRegister(PPC::R1, MVT::i32); PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, DAG.getConstant(ArgOffset, dl, PtrVT)); } MemOpChains.push_back( DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo())); // Calculate and remember argument location. } else CalculateTailCallArgDest(DAG, MF, isPPC64, Arg, SPDiff, ArgOffset, TailCallArguments); } static void PrepareTailCall(SelectionDAG &DAG, SDValue &InFlag, SDValue &Chain, const SDLoc &dl, int SPDiff, unsigned NumBytes, SDValue LROp, SDValue FPOp, SmallVectorImpl &TailCallArguments) { // Emit a sequence of copyto/copyfrom virtual registers for arguments that // might overwrite each other in case of tail call optimization. SmallVector MemOpChains2; // Do not flag preceding copytoreg stuff together with the following stuff. InFlag = SDValue(); StoreTailCallArgumentsToStackSlot(DAG, Chain, TailCallArguments, MemOpChains2, dl); if (!MemOpChains2.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2); // Store the return address to the appropriate stack slot. Chain = EmitTailCallStoreFPAndRetAddr(DAG, Chain, LROp, FPOp, SPDiff, dl); // Emit callseq_end just before tailcall node. Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), DAG.getIntPtrConstant(0, dl, true), InFlag, dl); InFlag = Chain.getValue(1); } // Is this global address that of a function that can be called by name? (as // opposed to something that must hold a descriptor for an indirect call). static bool isFunctionGlobalAddress(SDValue Callee) { if (GlobalAddressSDNode *G = dyn_cast(Callee)) { if (Callee.getOpcode() == ISD::GlobalTLSAddress || Callee.getOpcode() == ISD::TargetGlobalTLSAddress) return false; return G->getGlobal()->getValueType()->isFunctionTy(); } return false; } static unsigned PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag, SDValue &Chain, SDValue CallSeqStart, const SDLoc &dl, int SPDiff, bool isTailCall, bool isPatchPoint, bool hasNest, SmallVectorImpl> &RegsToPass, SmallVectorImpl &Ops, std::vector &NodeTys, ImmutableCallSite CS, const PPCSubtarget &Subtarget) { bool isPPC64 = Subtarget.isPPC64(); bool isSVR4ABI = Subtarget.isSVR4ABI(); bool isELFv2ABI = Subtarget.isELFv2ABI(); bool isAIXABI = Subtarget.isAIXABI(); EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); NodeTys.push_back(MVT::Other); // Returns a chain NodeTys.push_back(MVT::Glue); // Returns a flag for retval copy to use. unsigned CallOpc = PPCISD::CALL; bool needIndirectCall = true; if (!isSVR4ABI || !isPPC64) if (SDNode *Dest = isBLACompatibleAddress(Callee, DAG)) { // If this is an absolute destination address, use the munged value. Callee = SDValue(Dest, 0); needIndirectCall = false; } // PC-relative references to external symbols should go through $stub, unless // we're building with the leopard linker or later, which automatically // synthesizes these stubs. const TargetMachine &TM = DAG.getTarget(); const Module *Mod = DAG.getMachineFunction().getFunction().getParent(); const GlobalValue *GV = nullptr; if (auto *G = dyn_cast(Callee)) GV = G->getGlobal(); bool Local = TM.shouldAssumeDSOLocal(*Mod, GV); - bool UsePlt = !Local && Subtarget.isTargetELF() && !isPPC64; + // The PLT is only used in 32-bit ELF PIC mode. Attempting to use the PLT in + // a static relocation model causes some versions of GNU LD (2.17.50, at + // least) to force BSS-PLT, instead of secure-PLT, even if all objects are + // built with secure-PLT. + bool UsePlt = !Local && Subtarget.isTargetELF() && !isPPC64 && + Subtarget.getTargetMachine().getRelocationModel() == Reloc::PIC_; // If the callee is a GlobalAddress/ExternalSymbol node (quite common, // every direct call is) turn it into a TargetGlobalAddress / // TargetExternalSymbol node so that legalize doesn't hack it. if (isFunctionGlobalAddress(Callee)) { GlobalAddressSDNode *G = cast(Callee); // A call to a TLS address is actually an indirect call to a // thread-specific pointer. unsigned OpFlags = 0; if (UsePlt) OpFlags = PPCII::MO_PLT; Callee = DAG.getTargetGlobalAddress(G->getGlobal(), dl, Callee.getValueType(), 0, OpFlags); needIndirectCall = false; } if (ExternalSymbolSDNode *S = dyn_cast(Callee)) { unsigned char OpFlags = 0; if (UsePlt) OpFlags = PPCII::MO_PLT; Callee = DAG.getTargetExternalSymbol(S->getSymbol(), Callee.getValueType(), OpFlags); needIndirectCall = false; } if (isPatchPoint) { // We'll form an invalid direct call when lowering a patchpoint; the full // sequence for an indirect call is complicated, and many of the // instructions introduced might have side effects (and, thus, can't be // removed later). The call itself will be removed as soon as the // argument/return lowering is complete, so the fact that it has the wrong // kind of operands should not really matter. needIndirectCall = false; } if (needIndirectCall) { // Otherwise, this is an indirect call. We have to use a MTCTR/BCTRL pair // to do the call, we can't use PPCISD::CALL. SDValue MTCTROps[] = {Chain, Callee, InFlag}; if (isSVR4ABI && isPPC64 && !isELFv2ABI) { // Function pointers in the 64-bit SVR4 ABI do not point to the function // entry point, but to the function descriptor (the function entry point // address is part of the function descriptor though). // The function descriptor is a three doubleword structure with the // following fields: function entry point, TOC base address and // environment pointer. // Thus for a call through a function pointer, the following actions need // to be performed: // 1. Save the TOC of the caller in the TOC save area of its stack // frame (this is done in LowerCall_Darwin() or LowerCall_64SVR4()). // 2. Load the address of the function entry point from the function // descriptor. // 3. Load the TOC of the callee from the function descriptor into r2. // 4. Load the environment pointer from the function descriptor into // r11. // 5. Branch to the function entry point address. // 6. On return of the callee, the TOC of the caller needs to be // restored (this is done in FinishCall()). // // The loads are scheduled at the beginning of the call sequence, and the // register copies are flagged together to ensure that no other // operations can be scheduled in between. E.g. without flagging the // copies together, a TOC access in the caller could be scheduled between // the assignment of the callee TOC and the branch to the callee, which // results in the TOC access going through the TOC of the callee instead // of going through the TOC of the caller, which leads to incorrect code. // Load the address of the function entry point from the function // descriptor. SDValue LDChain = CallSeqStart.getValue(CallSeqStart->getNumValues()-1); if (LDChain.getValueType() == MVT::Glue) LDChain = CallSeqStart.getValue(CallSeqStart->getNumValues()-2); auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors() ? (MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant) : MachineMemOperand::MONone; MachinePointerInfo MPI(CS ? CS.getCalledValue() : nullptr); SDValue LoadFuncPtr = DAG.getLoad(MVT::i64, dl, LDChain, Callee, MPI, /* Alignment = */ 8, MMOFlags); // Load environment pointer into r11. SDValue PtrOff = DAG.getIntPtrConstant(16, dl); SDValue AddPtr = DAG.getNode(ISD::ADD, dl, MVT::i64, Callee, PtrOff); SDValue LoadEnvPtr = DAG.getLoad(MVT::i64, dl, LDChain, AddPtr, MPI.getWithOffset(16), /* Alignment = */ 8, MMOFlags); SDValue TOCOff = DAG.getIntPtrConstant(8, dl); SDValue AddTOC = DAG.getNode(ISD::ADD, dl, MVT::i64, Callee, TOCOff); SDValue TOCPtr = DAG.getLoad(MVT::i64, dl, LDChain, AddTOC, MPI.getWithOffset(8), /* Alignment = */ 8, MMOFlags); setUsesTOCBasePtr(DAG); SDValue TOCVal = DAG.getCopyToReg(Chain, dl, PPC::X2, TOCPtr, InFlag); Chain = TOCVal.getValue(0); InFlag = TOCVal.getValue(1); // If the function call has an explicit 'nest' parameter, it takes the // place of the environment pointer. if (!hasNest) { SDValue EnvVal = DAG.getCopyToReg(Chain, dl, PPC::X11, LoadEnvPtr, InFlag); Chain = EnvVal.getValue(0); InFlag = EnvVal.getValue(1); } MTCTROps[0] = Chain; MTCTROps[1] = LoadFuncPtr; MTCTROps[2] = InFlag; } Chain = DAG.getNode(PPCISD::MTCTR, dl, NodeTys, makeArrayRef(MTCTROps, InFlag.getNode() ? 3 : 2)); InFlag = Chain.getValue(1); NodeTys.clear(); NodeTys.push_back(MVT::Other); NodeTys.push_back(MVT::Glue); Ops.push_back(Chain); CallOpc = PPCISD::BCTRL; Callee.setNode(nullptr); // Add use of X11 (holding environment pointer) if (isSVR4ABI && isPPC64 && !isELFv2ABI && !hasNest) Ops.push_back(DAG.getRegister(PPC::X11, PtrVT)); // Add CTR register as callee so a bctr can be emitted later. if (isTailCall) Ops.push_back(DAG.getRegister(isPPC64 ? PPC::CTR8 : PPC::CTR, PtrVT)); } // If this is a direct call, pass the chain and the callee. if (Callee.getNode()) { Ops.push_back(Chain); Ops.push_back(Callee); } // If this is a tail call add stack pointer delta. if (isTailCall) Ops.push_back(DAG.getConstant(SPDiff, dl, MVT::i32)); // Add argument registers to the end of the list so that they are known live // into the call. for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) Ops.push_back(DAG.getRegister(RegsToPass[i].first, RegsToPass[i].second.getValueType())); // All calls, in the AIX ABI and 64-bit ELF ABIs, need the TOC register // live into the call. // We do need to reserve R2/X2 to appease the verifier for the PATCHPOINT. if ((isSVR4ABI && isPPC64) || isAIXABI) { setUsesTOCBasePtr(DAG); // We cannot add R2/X2 as an operand here for PATCHPOINT, because there is // no way to mark dependencies as implicit here. // We will add the R2/X2 dependency in EmitInstrWithCustomInserter. if (!isPatchPoint) Ops.push_back(DAG.getRegister(isPPC64 ? PPC::X2 : PPC::R2, PtrVT)); } return CallOpc; } SDValue PPCTargetLowering::LowerCallResult( SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const { SmallVector RVLocs; CCState CCRetInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, *DAG.getContext()); CCRetInfo.AnalyzeCallResult( Ins, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold) ? RetCC_PPC_Cold : RetCC_PPC); // Copy all of the result registers out of their specified physreg. for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) { CCValAssign &VA = RVLocs[i]; assert(VA.isRegLoc() && "Can only return in registers!"); SDValue Val; if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) { SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag); Chain = Lo.getValue(1); InFlag = Lo.getValue(2); VA = RVLocs[++i]; // skip ahead to next loc SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag); Chain = Hi.getValue(1); InFlag = Hi.getValue(2); if (!Subtarget.isLittleEndian()) std::swap (Lo, Hi); Val = DAG.getNode(PPCISD::BUILD_SPE64, dl, MVT::f64, Lo, Hi); } else { Val = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), VA.getLocVT(), InFlag); Chain = Val.getValue(1); InFlag = Val.getValue(2); } switch (VA.getLocInfo()) { default: llvm_unreachable("Unknown loc info!"); case CCValAssign::Full: break; case CCValAssign::AExt: Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); break; case CCValAssign::ZExt: Val = DAG.getNode(ISD::AssertZext, dl, VA.getLocVT(), Val, DAG.getValueType(VA.getValVT())); Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); break; case CCValAssign::SExt: Val = DAG.getNode(ISD::AssertSext, dl, VA.getLocVT(), Val, DAG.getValueType(VA.getValVT())); Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); break; } InVals.push_back(Val); } return Chain; } SDValue PPCTargetLowering::FinishCall( CallingConv::ID CallConv, const SDLoc &dl, bool isTailCall, bool isVarArg, bool isPatchPoint, bool hasNest, SelectionDAG &DAG, SmallVector, 8> &RegsToPass, SDValue InFlag, SDValue Chain, SDValue CallSeqStart, SDValue &Callee, int SPDiff, unsigned NumBytes, const SmallVectorImpl &Ins, SmallVectorImpl &InVals, ImmutableCallSite CS) const { std::vector NodeTys; SmallVector Ops; unsigned CallOpc = PrepareCall(DAG, Callee, InFlag, Chain, CallSeqStart, dl, SPDiff, isTailCall, isPatchPoint, hasNest, RegsToPass, Ops, NodeTys, CS, Subtarget); // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls if (isVarArg && Subtarget.isSVR4ABI() && !Subtarget.isPPC64()) Ops.push_back(DAG.getRegister(PPC::CR1EQ, MVT::i32)); // When performing tail call optimization the callee pops its arguments off // the stack. Account for this here so these bytes can be pushed back on in // PPCFrameLowering::eliminateCallFramePseudoInstr. int BytesCalleePops = (CallConv == CallingConv::Fast && getTargetMachine().Options.GuaranteedTailCallOpt) ? NumBytes : 0; // Add a register mask operand representing the call-preserved registers. const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); const uint32_t *Mask = TRI->getCallPreservedMask(DAG.getMachineFunction(), CallConv); assert(Mask && "Missing call preserved mask for calling convention"); Ops.push_back(DAG.getRegisterMask(Mask)); if (InFlag.getNode()) Ops.push_back(InFlag); // Emit tail call. if (isTailCall) { assert(((Callee.getOpcode() == ISD::Register && cast(Callee)->getReg() == PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol || Callee.getOpcode() == ISD::TargetGlobalAddress || isa(Callee)) && "Expecting an global address, external symbol, absolute value or register"); DAG.getMachineFunction().getFrameInfo().setHasTailCall(); return DAG.getNode(PPCISD::TC_RETURN, dl, MVT::Other, Ops); } // Add a NOP immediately after the branch instruction when using the 64-bit // SVR4 or the AIX ABI. // At link time, if caller and callee are in a different module and // thus have a different TOC, the call will be replaced with a call to a stub // function which saves the current TOC, loads the TOC of the callee and // branches to the callee. The NOP will be replaced with a load instruction // which restores the TOC of the caller from the TOC save slot of the current // stack frame. If caller and callee belong to the same module (and have the // same TOC), the NOP will remain unchanged, or become some other NOP. MachineFunction &MF = DAG.getMachineFunction(); EVT PtrVT = getPointerTy(DAG.getDataLayout()); if (!isTailCall && !isPatchPoint && ((Subtarget.isSVR4ABI() && Subtarget.isPPC64()) || Subtarget.isAIXABI())) { if (CallOpc == PPCISD::BCTRL) { if (Subtarget.isAIXABI()) report_fatal_error("Indirect call on AIX is not implemented."); // This is a call through a function pointer. // Restore the caller TOC from the save area into R2. // See PrepareCall() for more information about calls through function // pointers in the 64-bit SVR4 ABI. // We are using a target-specific load with r2 hard coded, because the // result of a target-independent load would never go directly into r2, // since r2 is a reserved register (which prevents the register allocator // from allocating it), resulting in an additional register being // allocated and an unnecessary move instruction being generated. CallOpc = PPCISD::BCTRL_LOAD_TOC; SDValue StackPtr = DAG.getRegister(PPC::X1, PtrVT); unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset(); SDValue TOCOff = DAG.getIntPtrConstant(TOCSaveOffset, dl); SDValue AddTOC = DAG.getNode(ISD::ADD, dl, MVT::i64, StackPtr, TOCOff); // The address needs to go after the chain input but before the flag (or // any other variadic arguments). Ops.insert(std::next(Ops.begin()), AddTOC); } else if (CallOpc == PPCISD::CALL && !callsShareTOCBase(&MF.getFunction(), Callee, DAG.getTarget())) { // Otherwise insert NOP for non-local calls. CallOpc = PPCISD::CALL_NOP; } } if (Subtarget.isAIXABI() && isFunctionGlobalAddress(Callee)) { // On AIX, direct function calls reference the symbol for the function's // entry point, which is named by inserting a "." before the function's // C-linkage name. GlobalAddressSDNode *G = cast(Callee); auto &Context = DAG.getMachineFunction().getMMI().getContext(); MCSymbol *S = Context.getOrCreateSymbol(Twine(".") + Twine(G->getGlobal()->getName())); Callee = DAG.getMCSymbol(S, PtrVT); // Replace the GlobalAddressSDNode Callee with the MCSymbolSDNode. Ops[1] = Callee; } Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops); InFlag = Chain.getValue(1); Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), DAG.getIntPtrConstant(BytesCalleePops, dl, true), InFlag, dl); if (!Ins.empty()) InFlag = Chain.getValue(1); return LowerCallResult(Chain, InFlag, CallConv, isVarArg, Ins, dl, DAG, InVals); } SDValue PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl &InVals) const { SelectionDAG &DAG = CLI.DAG; SDLoc &dl = CLI.DL; SmallVectorImpl &Outs = CLI.Outs; SmallVectorImpl &OutVals = CLI.OutVals; SmallVectorImpl &Ins = CLI.Ins; SDValue Chain = CLI.Chain; SDValue Callee = CLI.Callee; bool &isTailCall = CLI.IsTailCall; CallingConv::ID CallConv = CLI.CallConv; bool isVarArg = CLI.IsVarArg; bool isPatchPoint = CLI.IsPatchPoint; ImmutableCallSite CS = CLI.CS; if (isTailCall) { if (Subtarget.useLongCalls() && !(CS && CS.isMustTailCall())) isTailCall = false; else if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) isTailCall = IsEligibleForTailCallOptimization_64SVR4(Callee, CallConv, CS, isVarArg, Outs, Ins, DAG); else isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, isVarArg, Ins, DAG); if (isTailCall) { ++NumTailCalls; if (!getTargetMachine().Options.GuaranteedTailCallOpt) ++NumSiblingCalls; assert(isa(Callee) && "Callee should be an llvm::Function object."); LLVM_DEBUG( const GlobalValue *GV = cast(Callee)->getGlobal(); const unsigned Width = 80 - strlen("TCO caller: ") - strlen(", callee linkage: 0, 0"); dbgs() << "TCO caller: " << left_justify(DAG.getMachineFunction().getName(), Width) << ", callee linkage: " << GV->getVisibility() << ", " << GV->getLinkage() << "\n"); } } if (!isTailCall && CS && CS.isMustTailCall()) report_fatal_error("failed to perform tail call elimination on a call " "site marked musttail"); // When long calls (i.e. indirect calls) are always used, calls are always // made via function pointer. If we have a function name, first translate it // into a pointer. if (Subtarget.useLongCalls() && isa(Callee) && !isTailCall) Callee = LowerGlobalAddress(Callee, DAG); if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) return LowerCall_64SVR4(Chain, Callee, CallConv, isVarArg, isTailCall, isPatchPoint, Outs, OutVals, Ins, dl, DAG, InVals, CS); if (Subtarget.isSVR4ABI()) return LowerCall_32SVR4(Chain, Callee, CallConv, isVarArg, isTailCall, isPatchPoint, Outs, OutVals, Ins, dl, DAG, InVals, CS); if (Subtarget.isAIXABI()) return LowerCall_AIX(Chain, Callee, CallConv, isVarArg, isTailCall, isPatchPoint, Outs, OutVals, Ins, dl, DAG, InVals, CS); return LowerCall_Darwin(Chain, Callee, CallConv, isVarArg, isTailCall, isPatchPoint, Outs, OutVals, Ins, dl, DAG, InVals, CS); } SDValue PPCTargetLowering::LowerCall_32SVR4( SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg, bool isTailCall, bool isPatchPoint, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals, ImmutableCallSite CS) const { // See PPCTargetLowering::LowerFormalArguments_32SVR4() for a description // of the 32-bit SVR4 ABI stack frame layout. assert((CallConv == CallingConv::C || CallConv == CallingConv::Cold || CallConv == CallingConv::Fast) && "Unknown calling convention!"); unsigned PtrByteSize = 4; MachineFunction &MF = DAG.getMachineFunction(); // Mark this function as potentially containing a function that contains a // tail call. As a consequence the frame pointer will be used for dynamicalloc // and restoring the callers stack pointer in this functions epilog. This is // done because by tail calling the called function might overwrite the value // in this function's (MF) stack pointer stack slot 0(SP). if (getTargetMachine().Options.GuaranteedTailCallOpt && CallConv == CallingConv::Fast) MF.getInfo()->setHasFastCall(); // Count how many bytes are to be pushed on the stack, including the linkage // area, parameter list area and the part of the local variable space which // contains copies of aggregates which are passed by value. // Assign locations to all of the outgoing arguments. SmallVector ArgLocs; PPCCCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); // Reserve space for the linkage area on the stack. CCInfo.AllocateStack(Subtarget.getFrameLowering()->getLinkageSize(), PtrByteSize); if (useSoftFloat()) CCInfo.PreAnalyzeCallOperands(Outs); if (isVarArg) { // Handle fixed and variable vector arguments differently. // Fixed vector arguments go into registers as long as registers are // available. Variable vector arguments always go into memory. unsigned NumArgs = Outs.size(); for (unsigned i = 0; i != NumArgs; ++i) { MVT ArgVT = Outs[i].VT; ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; bool Result; if (Outs[i].IsFixed) { Result = CC_PPC32_SVR4(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo); } else { Result = CC_PPC32_SVR4_VarArg(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo); } if (Result) { #ifndef NDEBUG errs() << "Call operand #" << i << " has unhandled type " << EVT(ArgVT).getEVTString() << "\n"; #endif llvm_unreachable(nullptr); } } } else { // All arguments are treated the same. CCInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4); } CCInfo.clearWasPPCF128(); // Assign locations to all of the outgoing aggregate by value arguments. SmallVector ByValArgLocs; CCState CCByValInfo(CallConv, isVarArg, MF, ByValArgLocs, *DAG.getContext()); // Reserve stack space for the allocations in CCInfo. CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrByteSize); CCByValInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4_ByVal); // Size of the linkage area, parameter list area and the part of the local // space variable where copies of aggregates which are passed by value are // stored. unsigned NumBytes = CCByValInfo.getNextStackOffset(); // Calculate by how many bytes the stack has to be adjusted in case of tail // call optimization. int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes); // Adjust the stack pointer for the new arguments... // These operations are automatically eliminated by the prolog/epilog pass Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); SDValue CallSeqStart = Chain; // Load the return address and frame pointer so it can be moved somewhere else // later. SDValue LROp, FPOp; Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl); // Set up a copy of the stack pointer for use loading and storing any // arguments that may not fit in the registers available for argument // passing. SDValue StackPtr = DAG.getRegister(PPC::R1, MVT::i32); SmallVector, 8> RegsToPass; SmallVector TailCallArguments; SmallVector MemOpChains; bool seenFloatArg = false; // Walk the register/memloc assignments, inserting copies/loads. // i - Tracks the index into the list of registers allocated for the call // RealArgIdx - Tracks the index into the list of actual function arguments // j - Tracks the index into the list of byval arguments for (unsigned i = 0, RealArgIdx = 0, j = 0, e = ArgLocs.size(); i != e; ++i, ++RealArgIdx) { CCValAssign &VA = ArgLocs[i]; SDValue Arg = OutVals[RealArgIdx]; ISD::ArgFlagsTy Flags = Outs[RealArgIdx].Flags; if (Flags.isByVal()) { // Argument is an aggregate which is passed by value, thus we need to // create a copy of it in the local variable space of the current stack // frame (which is the stack frame of the caller) and pass the address of // this copy to the callee. assert((j < ByValArgLocs.size()) && "Index out of bounds!"); CCValAssign &ByValVA = ByValArgLocs[j++]; assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!"); // Memory reserved in the local variable space of the callers stack frame. unsigned LocMemOffset = ByValVA.getLocMemOffset(); SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()), StackPtr, PtrOff); // Create a copy of the argument in the local area of the current // stack frame. SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, PtrOff, CallSeqStart.getNode()->getOperand(0), Flags, DAG, dl); // This must go outside the CALLSEQ_START..END. SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, NumBytes, 0, SDLoc(MemcpyCall)); DAG.ReplaceAllUsesWith(CallSeqStart.getNode(), NewCallSeqStart.getNode()); Chain = CallSeqStart = NewCallSeqStart; // Pass the address of the aggregate copy on the stack either in a // physical register or in the parameter list area of the current stack // frame to the callee. Arg = PtrOff; } // When useCRBits() is true, there can be i1 arguments. // It is because getRegisterType(MVT::i1) => MVT::i1, // and for other integer types getRegisterType() => MVT::i32. // Extend i1 and ensure callee will get i32. if (Arg.getValueType() == MVT::i1) Arg = DAG.getNode(Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, MVT::i32, Arg); if (VA.isRegLoc()) { seenFloatArg |= VA.getLocVT().isFloatingPoint(); // Put argument in a physical register. if (Subtarget.hasSPE() && Arg.getValueType() == MVT::f64) { bool IsLE = Subtarget.isLittleEndian(); SDValue SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, DAG.getIntPtrConstant(IsLE ? 0 : 1, dl)); RegsToPass.push_back(std::make_pair(VA.getLocReg(), SVal.getValue(0))); SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, DAG.getIntPtrConstant(IsLE ? 1 : 0, dl)); RegsToPass.push_back(std::make_pair(ArgLocs[++i].getLocReg(), SVal.getValue(0))); } else RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); } else { // Put argument in the parameter list area of the current stack frame. assert(VA.isMemLoc()); unsigned LocMemOffset = VA.getLocMemOffset(); if (!isTailCall) { SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()), StackPtr, PtrOff); MemOpChains.push_back( DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo())); } else { // Calculate and remember argument location. CalculateTailCallArgDest(DAG, MF, false, Arg, SPDiff, LocMemOffset, TailCallArguments); } } } if (!MemOpChains.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); // Build a sequence of copy-to-reg nodes chained together with token chain // and flag operands which copy the outgoing args into the appropriate regs. SDValue InFlag; for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, RegsToPass[i].second, InFlag); InFlag = Chain.getValue(1); } // Set CR bit 6 to true if this is a vararg call with floating args passed in // registers. if (isVarArg) { SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue); SDValue Ops[] = { Chain, InFlag }; Chain = DAG.getNode(seenFloatArg ? PPCISD::CR6SET : PPCISD::CR6UNSET, dl, VTs, makeArrayRef(Ops, InFlag.getNode() ? 2 : 1)); InFlag = Chain.getValue(1); } if (isTailCall) PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp, TailCallArguments); return FinishCall(CallConv, dl, isTailCall, isVarArg, isPatchPoint, /* unused except on PPC64 ELFv1 */ false, DAG, RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff, NumBytes, Ins, InVals, CS); } // Copy an argument into memory, being careful to do this outside the // call sequence for the call to which the argument belongs. SDValue PPCTargetLowering::createMemcpyOutsideCallSeq( SDValue Arg, SDValue PtrOff, SDValue CallSeqStart, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, const SDLoc &dl) const { SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, PtrOff, CallSeqStart.getNode()->getOperand(0), Flags, DAG, dl); // The MEMCPY must go outside the CALLSEQ_START..END. int64_t FrameSize = CallSeqStart.getConstantOperandVal(1); SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, FrameSize, 0, SDLoc(MemcpyCall)); DAG.ReplaceAllUsesWith(CallSeqStart.getNode(), NewCallSeqStart.getNode()); return NewCallSeqStart; } SDValue PPCTargetLowering::LowerCall_64SVR4( SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg, bool isTailCall, bool isPatchPoint, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals, ImmutableCallSite CS) const { bool isELFv2ABI = Subtarget.isELFv2ABI(); bool isLittleEndian = Subtarget.isLittleEndian(); unsigned NumOps = Outs.size(); bool hasNest = false; bool IsSibCall = false; EVT PtrVT = getPointerTy(DAG.getDataLayout()); unsigned PtrByteSize = 8; MachineFunction &MF = DAG.getMachineFunction(); if (isTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt) IsSibCall = true; // Mark this function as potentially containing a function that contains a // tail call. As a consequence the frame pointer will be used for dynamicalloc // and restoring the callers stack pointer in this functions epilog. This is // done because by tail calling the called function might overwrite the value // in this function's (MF) stack pointer stack slot 0(SP). if (getTargetMachine().Options.GuaranteedTailCallOpt && CallConv == CallingConv::Fast) MF.getInfo()->setHasFastCall(); assert(!(CallConv == CallingConv::Fast && isVarArg) && "fastcc not supported on varargs functions"); // Count how many bytes are to be pushed on the stack, including the linkage // area, and parameter passing area. On ELFv1, the linkage area is 48 bytes // reserved space for [SP][CR][LR][2 x unused][TOC]; on ELFv2, the linkage // area is 32 bytes reserved space for [SP][CR][LR][TOC]. unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); unsigned NumBytes = LinkageSize; unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; unsigned &QFPR_idx = FPR_idx; static const MCPhysReg GPR[] = { PPC::X3, PPC::X4, PPC::X5, PPC::X6, PPC::X7, PPC::X8, PPC::X9, PPC::X10, }; static const MCPhysReg VR[] = { PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 }; const unsigned NumGPRs = array_lengthof(GPR); const unsigned NumFPRs = useSoftFloat() ? 0 : 13; const unsigned NumVRs = array_lengthof(VR); const unsigned NumQFPRs = NumFPRs; // On ELFv2, we can avoid allocating the parameter area if all the arguments // can be passed to the callee in registers. // For the fast calling convention, there is another check below. // Note: We should keep consistent with LowerFormalArguments_64SVR4() bool HasParameterArea = !isELFv2ABI || isVarArg || CallConv == CallingConv::Fast; if (!HasParameterArea) { unsigned ParamAreaSize = NumGPRs * PtrByteSize; unsigned AvailableFPRs = NumFPRs; unsigned AvailableVRs = NumVRs; unsigned NumBytesTmp = NumBytes; for (unsigned i = 0; i != NumOps; ++i) { if (Outs[i].Flags.isNest()) continue; if (CalculateStackSlotUsed(Outs[i].VT, Outs[i].ArgVT, Outs[i].Flags, PtrByteSize, LinkageSize, ParamAreaSize, NumBytesTmp, AvailableFPRs, AvailableVRs, Subtarget.hasQPX())) HasParameterArea = true; } } // When using the fast calling convention, we don't provide backing for // arguments that will be in registers. unsigned NumGPRsUsed = 0, NumFPRsUsed = 0, NumVRsUsed = 0; // Avoid allocating parameter area for fastcc functions if all the arguments // can be passed in the registers. if (CallConv == CallingConv::Fast) HasParameterArea = false; // Add up all the space actually used. for (unsigned i = 0; i != NumOps; ++i) { ISD::ArgFlagsTy Flags = Outs[i].Flags; EVT ArgVT = Outs[i].VT; EVT OrigVT = Outs[i].ArgVT; if (Flags.isNest()) continue; if (CallConv == CallingConv::Fast) { if (Flags.isByVal()) { NumGPRsUsed += (Flags.getByValSize()+7)/8; if (NumGPRsUsed > NumGPRs) HasParameterArea = true; } else { switch (ArgVT.getSimpleVT().SimpleTy) { default: llvm_unreachable("Unexpected ValueType for argument!"); case MVT::i1: case MVT::i32: case MVT::i64: if (++NumGPRsUsed <= NumGPRs) continue; break; case MVT::v4i32: case MVT::v8i16: case MVT::v16i8: case MVT::v2f64: case MVT::v2i64: case MVT::v1i128: case MVT::f128: if (++NumVRsUsed <= NumVRs) continue; break; case MVT::v4f32: // When using QPX, this is handled like a FP register, otherwise, it // is an Altivec register. if (Subtarget.hasQPX()) { if (++NumFPRsUsed <= NumFPRs) continue; } else { if (++NumVRsUsed <= NumVRs) continue; } break; case MVT::f32: case MVT::f64: case MVT::v4f64: // QPX case MVT::v4i1: // QPX if (++NumFPRsUsed <= NumFPRs) continue; break; } HasParameterArea = true; } } /* Respect alignment of argument on the stack. */ unsigned Align = CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); NumBytes = ((NumBytes + Align - 1) / Align) * Align; NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); if (Flags.isInConsecutiveRegsLast()) NumBytes = ((NumBytes + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; } unsigned NumBytesActuallyUsed = NumBytes; // In the old ELFv1 ABI, // the prolog code of the callee may store up to 8 GPR argument registers to // the stack, allowing va_start to index over them in memory if its varargs. // Because we cannot tell if this is needed on the caller side, we have to // conservatively assume that it is needed. As such, make sure we have at // least enough stack space for the caller to store the 8 GPRs. // In the ELFv2 ABI, we allocate the parameter area iff a callee // really requires memory operands, e.g. a vararg function. if (HasParameterArea) NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize); else NumBytes = LinkageSize; // Tail call needs the stack to be aligned. if (getTargetMachine().Options.GuaranteedTailCallOpt && CallConv == CallingConv::Fast) NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes); int SPDiff = 0; // Calculate by how many bytes the stack has to be adjusted in case of tail // call optimization. if (!IsSibCall) SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes); // To protect arguments on the stack from being clobbered in a tail call, // force all the loads to happen before doing any other lowering. if (isTailCall) Chain = DAG.getStackArgumentTokenFactor(Chain); // Adjust the stack pointer for the new arguments... // These operations are automatically eliminated by the prolog/epilog pass if (!IsSibCall) Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); SDValue CallSeqStart = Chain; // Load the return address and frame pointer so it can be move somewhere else // later. SDValue LROp, FPOp; Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl); // Set up a copy of the stack pointer for use loading and storing any // arguments that may not fit in the registers available for argument // passing. SDValue StackPtr = DAG.getRegister(PPC::X1, MVT::i64); // Figure out which arguments are going to go in registers, and which in // memory. Also, if this is a vararg function, floating point operations // must be stored to our stack, and loaded into integer regs as well, if // any integer regs are available for argument passing. unsigned ArgOffset = LinkageSize; SmallVector, 8> RegsToPass; SmallVector TailCallArguments; SmallVector MemOpChains; for (unsigned i = 0; i != NumOps; ++i) { SDValue Arg = OutVals[i]; ISD::ArgFlagsTy Flags = Outs[i].Flags; EVT ArgVT = Outs[i].VT; EVT OrigVT = Outs[i].ArgVT; // PtrOff will be used to store the current argument to the stack if a // register cannot be found for it. SDValue PtrOff; // We re-align the argument offset for each argument, except when using the // fast calling convention, when we need to make sure we do that only when // we'll actually use a stack slot. auto ComputePtrOff = [&]() { /* Respect alignment of argument on the stack. */ unsigned Align = CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); ArgOffset = ((ArgOffset + Align - 1) / Align) * Align; PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType()); PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); }; if (CallConv != CallingConv::Fast) { ComputePtrOff(); /* Compute GPR index associated with argument offset. */ GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; GPR_idx = std::min(GPR_idx, NumGPRs); } // Promote integers to 64-bit values. if (Arg.getValueType() == MVT::i32 || Arg.getValueType() == MVT::i1) { // FIXME: Should this use ANY_EXTEND if neither sext nor zext? unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg); } // FIXME memcpy is used way more than necessary. Correctness first. // Note: "by value" is code for passing a structure by value, not // basic types. if (Flags.isByVal()) { // Note: Size includes alignment padding, so // struct x { short a; char b; } // will have Size = 4. With #pragma pack(1), it will have Size = 3. // These are the proper values we need for right-justifying the // aggregate in a parameter register. unsigned Size = Flags.getByValSize(); // An empty aggregate parameter takes up no storage and no // registers. if (Size == 0) continue; if (CallConv == CallingConv::Fast) ComputePtrOff(); // All aggregates smaller than 8 bytes must be passed right-justified. if (Size==1 || Size==2 || Size==4) { EVT VT = (Size==1) ? MVT::i8 : ((Size==2) ? MVT::i16 : MVT::i32); if (GPR_idx != NumGPRs) { SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg, MachinePointerInfo(), VT); MemOpChains.push_back(Load.getValue(1)); RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); ArgOffset += PtrByteSize; continue; } } if (GPR_idx == NumGPRs && Size < 8) { SDValue AddPtr = PtrOff; if (!isLittleEndian) { SDValue Const = DAG.getConstant(PtrByteSize - Size, dl, PtrOff.getValueType()); AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); } Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, CallSeqStart, Flags, DAG, dl); ArgOffset += PtrByteSize; continue; } // Copy entire object into memory. There are cases where gcc-generated // code assumes it is there, even if it could be put entirely into // registers. (This is not what the doc says.) // FIXME: The above statement is likely due to a misunderstanding of the // documents. All arguments must be copied into the parameter area BY // THE CALLEE in the event that the callee takes the address of any // formal argument. That has not yet been implemented. However, it is // reasonable to use the stack area as a staging area for the register // load. // Skip this for small aggregates, as we will use the same slot for a // right-justified copy, below. if (Size >= 8) Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff, CallSeqStart, Flags, DAG, dl); // When a register is available, pass a small aggregate right-justified. if (Size < 8 && GPR_idx != NumGPRs) { // The easiest way to get this right-justified in a register // is to copy the structure into the rightmost portion of a // local variable slot, then load the whole slot into the // register. // FIXME: The memcpy seems to produce pretty awful code for // small aggregates, particularly for packed ones. // FIXME: It would be preferable to use the slot in the // parameter save area instead of a new local variable. SDValue AddPtr = PtrOff; if (!isLittleEndian) { SDValue Const = DAG.getConstant(8 - Size, dl, PtrOff.getValueType()); AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); } Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, CallSeqStart, Flags, DAG, dl); // Load the slot into the register. SDValue Load = DAG.getLoad(PtrVT, dl, Chain, PtrOff, MachinePointerInfo()); MemOpChains.push_back(Load.getValue(1)); RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); // Done with this argument. ArgOffset += PtrByteSize; continue; } // For aggregates larger than PtrByteSize, copy the pieces of the // object that fit into registers from the parameter save area. for (unsigned j=0; j gpr moves. // In the non-vararg case, this can only ever happen in the // presence of f32 array types, since otherwise we never run // out of FPRs before running out of GPRs. SDValue ArgVal; // Double values are always passed in a single GPR. if (Arg.getValueType() != MVT::f32) { ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg); // Non-array float values are extended and passed in a GPR. } else if (!Flags.isInConsecutiveRegs()) { ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal); // If we have an array of floats, we collect every odd element // together with its predecessor into one GPR. } else if (ArgOffset % PtrByteSize != 0) { SDValue Lo, Hi; Lo = DAG.getNode(ISD::BITCAST, dl, MVT::i32, OutVals[i - 1]); Hi = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); if (!isLittleEndian) std::swap(Lo, Hi); ArgVal = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); // The final element, if even, goes into the first half of a GPR. } else if (Flags.isInConsecutiveRegsLast()) { ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal); if (!isLittleEndian) ArgVal = DAG.getNode(ISD::SHL, dl, MVT::i64, ArgVal, DAG.getConstant(32, dl, MVT::i32)); // Non-final even elements are skipped; they will be handled // together the with subsequent argument on the next go-around. } else ArgVal = SDValue(); if (ArgVal.getNode()) RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], ArgVal)); } else { if (CallConv == CallingConv::Fast) ComputePtrOff(); // Single-precision floating-point values are mapped to the // second (rightmost) word of the stack doubleword. if (Arg.getValueType() == MVT::f32 && !isLittleEndian && !Flags.isInConsecutiveRegs()) { SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType()); PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour); } assert(HasParameterArea && "Parameter area must exist to pass an argument in memory."); LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, true, isTailCall, false, MemOpChains, TailCallArguments, dl); NeededLoad = true; } // When passing an array of floats, the array occupies consecutive // space in the argument area; only round up to the next doubleword // at the end of the array. Otherwise, each float takes 8 bytes. if (CallConv != CallingConv::Fast || NeededLoad) { ArgOffset += (Arg.getValueType() == MVT::f32 && Flags.isInConsecutiveRegs()) ? 4 : 8; if (Flags.isInConsecutiveRegsLast()) ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; } break; } case MVT::v4f32: case MVT::v4i32: case MVT::v8i16: case MVT::v16i8: case MVT::v2f64: case MVT::v2i64: case MVT::v1i128: case MVT::f128: if (!Subtarget.hasQPX()) { // These can be scalar arguments or elements of a vector array type // passed directly. The latter are used to implement ELFv2 homogenous // vector aggregates. // For a varargs call, named arguments go into VRs or on the stack as // usual; unnamed arguments always go to the stack or the corresponding // GPRs when within range. For now, we always put the value in both // locations (or even all three). if (isVarArg) { assert(HasParameterArea && "Parameter area must exist if we have a varargs call."); // We could elide this store in the case where the object fits // entirely in R registers. Maybe later. SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); MemOpChains.push_back(Store); if (VR_idx != NumVRs) { SDValue Load = DAG.getLoad(MVT::v4f32, dl, Store, PtrOff, MachinePointerInfo()); MemOpChains.push_back(Load.getValue(1)); RegsToPass.push_back(std::make_pair(VR[VR_idx++], Load)); } ArgOffset += 16; for (unsigned i=0; i<16; i+=PtrByteSize) { if (GPR_idx == NumGPRs) break; SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, DAG.getConstant(i, dl, PtrVT)); SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo()); MemOpChains.push_back(Load.getValue(1)); RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); } break; } // Non-varargs Altivec params go into VRs or on the stack. if (VR_idx != NumVRs) { RegsToPass.push_back(std::make_pair(VR[VR_idx++], Arg)); } else { if (CallConv == CallingConv::Fast) ComputePtrOff(); assert(HasParameterArea && "Parameter area must exist to pass an argument in memory."); LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, true, isTailCall, true, MemOpChains, TailCallArguments, dl); if (CallConv == CallingConv::Fast) ArgOffset += 16; } if (CallConv != CallingConv::Fast) ArgOffset += 16; break; } // not QPX assert(Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32 && "Invalid QPX parameter type"); LLVM_FALLTHROUGH; case MVT::v4f64: case MVT::v4i1: { bool IsF32 = Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32; if (isVarArg) { assert(HasParameterArea && "Parameter area must exist if we have a varargs call."); // We could elide this store in the case where the object fits // entirely in R registers. Maybe later. SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); MemOpChains.push_back(Store); if (QFPR_idx != NumQFPRs) { SDValue Load = DAG.getLoad(IsF32 ? MVT::v4f32 : MVT::v4f64, dl, Store, PtrOff, MachinePointerInfo()); MemOpChains.push_back(Load.getValue(1)); RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Load)); } ArgOffset += (IsF32 ? 16 : 32); for (unsigned i = 0; i < (IsF32 ? 16U : 32U); i += PtrByteSize) { if (GPR_idx == NumGPRs) break; SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, DAG.getConstant(i, dl, PtrVT)); SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo()); MemOpChains.push_back(Load.getValue(1)); RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); } break; } // Non-varargs QPX params go into registers or on the stack. if (QFPR_idx != NumQFPRs) { RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Arg)); } else { if (CallConv == CallingConv::Fast) ComputePtrOff(); assert(HasParameterArea && "Parameter area must exist to pass an argument in memory."); LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, true, isTailCall, true, MemOpChains, TailCallArguments, dl); if (CallConv == CallingConv::Fast) ArgOffset += (IsF32 ? 16 : 32); } if (CallConv != CallingConv::Fast) ArgOffset += (IsF32 ? 16 : 32); break; } } } assert((!HasParameterArea || NumBytesActuallyUsed == ArgOffset) && "mismatch in size of parameter area"); (void)NumBytesActuallyUsed; if (!MemOpChains.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); // Check if this is an indirect call (MTCTR/BCTRL). // See PrepareCall() for more information about calls through function // pointers in the 64-bit SVR4 ABI. if (!isTailCall && !isPatchPoint && !isFunctionGlobalAddress(Callee) && !isa(Callee)) { // Load r2 into a virtual register and store it to the TOC save area. setUsesTOCBasePtr(DAG); SDValue Val = DAG.getCopyFromReg(Chain, dl, PPC::X2, MVT::i64); // TOC save area offset. unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset(); SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl); SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); Chain = DAG.getStore( Val.getValue(1), dl, Val, AddPtr, MachinePointerInfo::getStack(DAG.getMachineFunction(), TOCSaveOffset)); // In the ELFv2 ABI, R12 must contain the address of an indirect callee. // This does not mean the MTCTR instruction must use R12; it's easier // to model this as an extra parameter, so do that. if (isELFv2ABI && !isPatchPoint) RegsToPass.push_back(std::make_pair((unsigned)PPC::X12, Callee)); } // Build a sequence of copy-to-reg nodes chained together with token chain // and flag operands which copy the outgoing args into the appropriate regs. SDValue InFlag; for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, RegsToPass[i].second, InFlag); InFlag = Chain.getValue(1); } if (isTailCall && !IsSibCall) PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp, TailCallArguments); return FinishCall(CallConv, dl, isTailCall, isVarArg, isPatchPoint, hasNest, DAG, RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff, NumBytes, Ins, InVals, CS); } SDValue PPCTargetLowering::LowerCall_Darwin( SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg, bool isTailCall, bool isPatchPoint, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals, ImmutableCallSite CS) const { unsigned NumOps = Outs.size(); EVT PtrVT = getPointerTy(DAG.getDataLayout()); bool isPPC64 = PtrVT == MVT::i64; unsigned PtrByteSize = isPPC64 ? 8 : 4; MachineFunction &MF = DAG.getMachineFunction(); // Mark this function as potentially containing a function that contains a // tail call. As a consequence the frame pointer will be used for dynamicalloc // and restoring the callers stack pointer in this functions epilog. This is // done because by tail calling the called function might overwrite the value // in this function's (MF) stack pointer stack slot 0(SP). if (getTargetMachine().Options.GuaranteedTailCallOpt && CallConv == CallingConv::Fast) MF.getInfo()->setHasFastCall(); // Count how many bytes are to be pushed on the stack, including the linkage // area, and parameter passing area. We start with 24/48 bytes, which is // prereserved space for [SP][CR][LR][3 x unused]. unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); unsigned NumBytes = LinkageSize; // Add up all the space actually used. // In 32-bit non-varargs calls, Altivec parameters all go at the end; usually // they all go in registers, but we must reserve stack space for them for // possible use by the caller. In varargs or 64-bit calls, parameters are // assigned stack space in order, with padding so Altivec parameters are // 16-byte aligned. unsigned nAltivecParamsAtEnd = 0; for (unsigned i = 0; i != NumOps; ++i) { ISD::ArgFlagsTy Flags = Outs[i].Flags; EVT ArgVT = Outs[i].VT; // Varargs Altivec parameters are padded to a 16 byte boundary. if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64) { if (!isVarArg && !isPPC64) { // Non-varargs Altivec parameters go after all the non-Altivec // parameters; handle those later so we know how much padding we need. nAltivecParamsAtEnd++; continue; } // Varargs and 64-bit Altivec parameters are padded to 16 byte boundary. NumBytes = ((NumBytes+15)/16)*16; } NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); } // Allow for Altivec parameters at the end, if needed. if (nAltivecParamsAtEnd) { NumBytes = ((NumBytes+15)/16)*16; NumBytes += 16*nAltivecParamsAtEnd; } // The prolog code of the callee may store up to 8 GPR argument registers to // the stack, allowing va_start to index over them in memory if its varargs. // Because we cannot tell if this is needed on the caller side, we have to // conservatively assume that it is needed. As such, make sure we have at // least enough stack space for the caller to store the 8 GPRs. NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize); // Tail call needs the stack to be aligned. if (getTargetMachine().Options.GuaranteedTailCallOpt && CallConv == CallingConv::Fast) NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes); // Calculate by how many bytes the stack has to be adjusted in case of tail // call optimization. int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes); // To protect arguments on the stack from being clobbered in a tail call, // force all the loads to happen before doing any other lowering. if (isTailCall) Chain = DAG.getStackArgumentTokenFactor(Chain); // Adjust the stack pointer for the new arguments... // These operations are automatically eliminated by the prolog/epilog pass Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); SDValue CallSeqStart = Chain; // Load the return address and frame pointer so it can be move somewhere else // later. SDValue LROp, FPOp; Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl); // Set up a copy of the stack pointer for use loading and storing any // arguments that may not fit in the registers available for argument // passing. SDValue StackPtr; if (isPPC64) StackPtr = DAG.getRegister(PPC::X1, MVT::i64); else StackPtr = DAG.getRegister(PPC::R1, MVT::i32); // Figure out which arguments are going to go in registers, and which in // memory. Also, if this is a vararg function, floating point operations // must be stored to our stack, and loaded into integer regs as well, if // any integer regs are available for argument passing. unsigned ArgOffset = LinkageSize; unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; static const MCPhysReg GPR_32[] = { // 32-bit registers. PPC::R3, PPC::R4, PPC::R5, PPC::R6, PPC::R7, PPC::R8, PPC::R9, PPC::R10, }; static const MCPhysReg GPR_64[] = { // 64-bit registers. PPC::X3, PPC::X4, PPC::X5, PPC::X6, PPC::X7, PPC::X8, PPC::X9, PPC::X10, }; static const MCPhysReg VR[] = { PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 }; const unsigned NumGPRs = array_lengthof(GPR_32); const unsigned NumFPRs = 13; const unsigned NumVRs = array_lengthof(VR); const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; SmallVector, 8> RegsToPass; SmallVector TailCallArguments; SmallVector MemOpChains; for (unsigned i = 0; i != NumOps; ++i) { SDValue Arg = OutVals[i]; ISD::ArgFlagsTy Flags = Outs[i].Flags; // PtrOff will be used to store the current argument to the stack if a // register cannot be found for it. SDValue PtrOff; PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType()); PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); // On PPC64, promote integers to 64-bit values. if (isPPC64 && Arg.getValueType() == MVT::i32) { // FIXME: Should this use ANY_EXTEND if neither sext nor zext? unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg); } // FIXME memcpy is used way more than necessary. Correctness first. // Note: "by value" is code for passing a structure by value, not // basic types. if (Flags.isByVal()) { unsigned Size = Flags.getByValSize(); // Very small objects are passed right-justified. Everything else is // passed left-justified. if (Size==1 || Size==2) { EVT VT = (Size==1) ? MVT::i8 : MVT::i16; if (GPR_idx != NumGPRs) { SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg, MachinePointerInfo(), VT); MemOpChains.push_back(Load.getValue(1)); RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); ArgOffset += PtrByteSize; } else { SDValue Const = DAG.getConstant(PtrByteSize - Size, dl, PtrOff.getValueType()); SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, CallSeqStart, Flags, DAG, dl); ArgOffset += PtrByteSize; } continue; } // Copy entire object into memory. There are cases where gcc-generated // code assumes it is there, even if it could be put entirely into // registers. (This is not what the doc says.) Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff, CallSeqStart, Flags, DAG, dl); // For small aggregates (Darwin only) and aggregates >= PtrByteSize, // copy the pieces of the object that fit into registers from the // parameter save area. for (unsigned j=0; j NumVRs) { unsigned j = 0; // Offset is aligned; skip 1st 12 params which go in V registers. ArgOffset = ((ArgOffset+15)/16)*16; ArgOffset += 12*16; for (unsigned i = 0; i != NumOps; ++i) { SDValue Arg = OutVals[i]; EVT ArgType = Outs[i].VT; if (ArgType==MVT::v4f32 || ArgType==MVT::v4i32 || ArgType==MVT::v8i16 || ArgType==MVT::v16i8) { if (++j > NumVRs) { SDValue PtrOff; // We are emitting Altivec params in order. LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, isPPC64, isTailCall, true, MemOpChains, TailCallArguments, dl); ArgOffset += 16; } } } } if (!MemOpChains.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); // On Darwin, R12 must contain the address of an indirect callee. This does // not mean the MTCTR instruction must use R12; it's easier to model this as // an extra parameter, so do that. if (!isTailCall && !isFunctionGlobalAddress(Callee) && !isa(Callee) && !isBLACompatibleAddress(Callee, DAG)) RegsToPass.push_back(std::make_pair((unsigned)(isPPC64 ? PPC::X12 : PPC::R12), Callee)); // Build a sequence of copy-to-reg nodes chained together with token chain // and flag operands which copy the outgoing args into the appropriate regs. SDValue InFlag; for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, RegsToPass[i].second, InFlag); InFlag = Chain.getValue(1); } if (isTailCall) PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp, TailCallArguments); return FinishCall(CallConv, dl, isTailCall, isVarArg, isPatchPoint, /* unused except on PPC64 ELFv1 */ false, DAG, RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff, NumBytes, Ins, InVals, CS); } SDValue PPCTargetLowering::LowerCall_AIX( SDValue Chain, SDValue Callee, CallingConv::ID CallConv, bool isVarArg, bool isTailCall, bool isPatchPoint, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals, ImmutableCallSite CS) const { assert((CallConv == CallingConv::C || CallConv == CallingConv::Fast) && "Unimplemented calling convention!"); if (isVarArg || isPatchPoint) report_fatal_error("This call type is unimplemented on AIX."); EVT PtrVT = getPointerTy(DAG.getDataLayout()); bool isPPC64 = PtrVT == MVT::i64; unsigned PtrByteSize = isPPC64 ? 8 : 4; unsigned NumOps = Outs.size(); // Count how many bytes are to be pushed on the stack, including the linkage // area, parameter list area. // On XCOFF, we start with 24/48, which is reserved space for // [SP][CR][LR][2 x reserved][TOC]. unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); // The prolog code of the callee may store up to 8 GPR argument registers to // the stack, allowing va_start to index over them in memory if the callee // is variadic. // Because we cannot tell if this is needed on the caller side, we have to // conservatively assume that it is needed. As such, make sure we have at // least enough stack space for the caller to store the 8 GPRs. unsigned NumBytes = LinkageSize + 8 * PtrByteSize; // Adjust the stack pointer for the new arguments... // These operations are automatically eliminated by the prolog/epilog // inserter pass. Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); SDValue CallSeqStart = Chain; static const MCPhysReg GPR_32[] = { // 32-bit registers. PPC::R3, PPC::R4, PPC::R5, PPC::R6, PPC::R7, PPC::R8, PPC::R9, PPC::R10 }; static const MCPhysReg GPR_64[] = { // 64-bit registers. PPC::X3, PPC::X4, PPC::X5, PPC::X6, PPC::X7, PPC::X8, PPC::X9, PPC::X10 }; const unsigned NumGPRs = isPPC64 ? array_lengthof(GPR_64) : array_lengthof(GPR_32); const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; unsigned GPR_idx = 0; SmallVector, 8> RegsToPass; if (isTailCall) report_fatal_error("Handling of tail call is unimplemented!"); int SPDiff = 0; for (unsigned i = 0; i != NumOps; ++i) { SDValue Arg = OutVals[i]; ISD::ArgFlagsTy Flags = Outs[i].Flags; // Promote integers if needed. if (Arg.getValueType() == MVT::i1 || (isPPC64 && Arg.getValueType() == MVT::i32)) { unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; Arg = DAG.getNode(ExtOp, dl, PtrVT, Arg); } // Note: "by value" is code for passing a structure by value, not // basic types. if (Flags.isByVal()) report_fatal_error("Passing structure by value is unimplemented!"); switch (Arg.getSimpleValueType().SimpleTy) { default: llvm_unreachable("Unexpected ValueType for argument!"); case MVT::i1: case MVT::i32: case MVT::i64: if (GPR_idx != NumGPRs) RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg)); else report_fatal_error("Handling of placing parameters on the stack is " "unimplemented!"); break; case MVT::f32: case MVT::f64: case MVT::v4f32: case MVT::v4i32: case MVT::v8i16: case MVT::v16i8: case MVT::v2f64: case MVT::v2i64: case MVT::v1i128: case MVT::f128: case MVT::v4f64: case MVT::v4i1: report_fatal_error("Handling of this parameter type is unimplemented!"); } } if (!isFunctionGlobalAddress(Callee) && !isa(Callee)) report_fatal_error("Handling of indirect call is unimplemented!"); // Build a sequence of copy-to-reg nodes chained together with token chain // and flag operands which copy the outgoing args into the appropriate regs. SDValue InFlag; for (auto Reg : RegsToPass) { Chain = DAG.getCopyToReg(Chain, dl, Reg.first, Reg.second, InFlag); InFlag = Chain.getValue(1); } return FinishCall(CallConv, dl, isTailCall, isVarArg, isPatchPoint, /* unused except on PPC64 ELFv1 */ false, DAG, RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff, NumBytes, Ins, InVals, CS); } bool PPCTargetLowering::CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg, const SmallVectorImpl &Outs, LLVMContext &Context) const { SmallVector RVLocs; CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context); return CCInfo.CheckReturn( Outs, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold) ? RetCC_PPC_Cold : RetCC_PPC); } SDValue PPCTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SDLoc &dl, SelectionDAG &DAG) const { SmallVector RVLocs; CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, *DAG.getContext()); CCInfo.AnalyzeReturn(Outs, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold) ? RetCC_PPC_Cold : RetCC_PPC); SDValue Flag; SmallVector RetOps(1, Chain); // Copy the result values into the output registers. for (unsigned i = 0, RealResIdx = 0; i != RVLocs.size(); ++i, ++RealResIdx) { CCValAssign &VA = RVLocs[i]; assert(VA.isRegLoc() && "Can only return in registers!"); SDValue Arg = OutVals[RealResIdx]; switch (VA.getLocInfo()) { default: llvm_unreachable("Unknown loc info!"); case CCValAssign::Full: break; case CCValAssign::AExt: Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg); break; case CCValAssign::ZExt: Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); break; case CCValAssign::SExt: Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); break; } if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) { bool isLittleEndian = Subtarget.isLittleEndian(); // Legalize ret f64 -> ret 2 x i32. SDValue SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, DAG.getIntPtrConstant(isLittleEndian ? 0 : 1, dl)); Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), SVal, Flag); RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, DAG.getIntPtrConstant(isLittleEndian ? 1 : 0, dl)); Flag = Chain.getValue(1); VA = RVLocs[++i]; // skip ahead to next loc Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), SVal, Flag); } else Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag); Flag = Chain.getValue(1); RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); } const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo(); const MCPhysReg *I = TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction()); if (I) { for (; *I; ++I) { if (PPC::G8RCRegClass.contains(*I)) RetOps.push_back(DAG.getRegister(*I, MVT::i64)); else if (PPC::F8RCRegClass.contains(*I)) RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64))); else if (PPC::CRRCRegClass.contains(*I)) RetOps.push_back(DAG.getRegister(*I, MVT::i1)); else if (PPC::VRRCRegClass.contains(*I)) RetOps.push_back(DAG.getRegister(*I, MVT::Other)); else llvm_unreachable("Unexpected register class in CSRsViaCopy!"); } } RetOps[0] = Chain; // Update chain. // Add the flag if we have it. if (Flag.getNode()) RetOps.push_back(Flag); return DAG.getNode(PPCISD::RET_FLAG, dl, MVT::Other, RetOps); } SDValue PPCTargetLowering::LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); // Get the correct type for integers. EVT IntVT = Op.getValueType(); // Get the inputs. SDValue Chain = Op.getOperand(0); SDValue FPSIdx = getFramePointerFrameIndex(DAG); // Build a DYNAREAOFFSET node. SDValue Ops[2] = {Chain, FPSIdx}; SDVTList VTs = DAG.getVTList(IntVT); return DAG.getNode(PPCISD::DYNAREAOFFSET, dl, VTs, Ops); } SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op, SelectionDAG &DAG) const { // When we pop the dynamic allocation we need to restore the SP link. SDLoc dl(Op); // Get the correct type for pointers. EVT PtrVT = getPointerTy(DAG.getDataLayout()); // Construct the stack pointer operand. bool isPPC64 = Subtarget.isPPC64(); unsigned SP = isPPC64 ? PPC::X1 : PPC::R1; SDValue StackPtr = DAG.getRegister(SP, PtrVT); // Get the operands for the STACKRESTORE. SDValue Chain = Op.getOperand(0); SDValue SaveSP = Op.getOperand(1); // Load the old link SP. SDValue LoadLinkSP = DAG.getLoad(PtrVT, dl, Chain, StackPtr, MachinePointerInfo()); // Restore the stack pointer. Chain = DAG.getCopyToReg(LoadLinkSP.getValue(1), dl, SP, SaveSP); // Store the old link SP. return DAG.getStore(Chain, dl, LoadLinkSP, StackPtr, MachinePointerInfo()); } SDValue PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG &DAG) const { MachineFunction &MF = DAG.getMachineFunction(); bool isPPC64 = Subtarget.isPPC64(); EVT PtrVT = getPointerTy(MF.getDataLayout()); // Get current frame pointer save index. The users of this index will be // primarily DYNALLOC instructions. PPCFunctionInfo *FI = MF.getInfo(); int RASI = FI->getReturnAddrSaveIndex(); // If the frame pointer save index hasn't been defined yet. if (!RASI) { // Find out what the fix offset of the frame pointer save area. int LROffset = Subtarget.getFrameLowering()->getReturnSaveOffset(); // Allocate the frame index for frame pointer save area. RASI = MF.getFrameInfo().CreateFixedObject(isPPC64? 8 : 4, LROffset, false); // Save the result. FI->setReturnAddrSaveIndex(RASI); } return DAG.getFrameIndex(RASI, PtrVT); } SDValue PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const { MachineFunction &MF = DAG.getMachineFunction(); bool isPPC64 = Subtarget.isPPC64(); EVT PtrVT = getPointerTy(MF.getDataLayout()); // Get current frame pointer save index. The users of this index will be // primarily DYNALLOC instructions. PPCFunctionInfo *FI = MF.getInfo(); int FPSI = FI->getFramePointerSaveIndex(); // If the frame pointer save index hasn't been defined yet. if (!FPSI) { // Find out what the fix offset of the frame pointer save area. int FPOffset = Subtarget.getFrameLowering()->getFramePointerSaveOffset(); // Allocate the frame index for frame pointer save area. FPSI = MF.getFrameInfo().CreateFixedObject(isPPC64? 8 : 4, FPOffset, true); // Save the result. FI->setFramePointerSaveIndex(FPSI); } return DAG.getFrameIndex(FPSI, PtrVT); } SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const { // Get the inputs. SDValue Chain = Op.getOperand(0); SDValue Size = Op.getOperand(1); SDLoc dl(Op); // Get the correct type for pointers. EVT PtrVT = getPointerTy(DAG.getDataLayout()); // Negate the size. SDValue NegSize = DAG.getNode(ISD::SUB, dl, PtrVT, DAG.getConstant(0, dl, PtrVT), Size); // Construct a node for the frame pointer save index. SDValue FPSIdx = getFramePointerFrameIndex(DAG); // Build a DYNALLOC node. SDValue Ops[3] = { Chain, NegSize, FPSIdx }; SDVTList VTs = DAG.getVTList(PtrVT, MVT::Other); return DAG.getNode(PPCISD::DYNALLOC, dl, VTs, Ops); } SDValue PPCTargetLowering::LowerEH_DWARF_CFA(SDValue Op, SelectionDAG &DAG) const { MachineFunction &MF = DAG.getMachineFunction(); bool isPPC64 = Subtarget.isPPC64(); EVT PtrVT = getPointerTy(DAG.getDataLayout()); int FI = MF.getFrameInfo().CreateFixedObject(isPPC64 ? 8 : 4, 0, false); return DAG.getFrameIndex(FI, PtrVT); } SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const { SDLoc DL(Op); return DAG.getNode(PPCISD::EH_SJLJ_SETJMP, DL, DAG.getVTList(MVT::i32, MVT::Other), Op.getOperand(0), Op.getOperand(1)); } SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const { SDLoc DL(Op); return DAG.getNode(PPCISD::EH_SJLJ_LONGJMP, DL, MVT::Other, Op.getOperand(0), Op.getOperand(1)); } SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { if (Op.getValueType().isVector()) return LowerVectorLoad(Op, DAG); assert(Op.getValueType() == MVT::i1 && "Custom lowering only for i1 loads"); // First, load 8 bits into 32 bits, then truncate to 1 bit. SDLoc dl(Op); LoadSDNode *LD = cast(Op); SDValue Chain = LD->getChain(); SDValue BasePtr = LD->getBasePtr(); MachineMemOperand *MMO = LD->getMemOperand(); SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, dl, getPointerTy(DAG.getDataLayout()), Chain, BasePtr, MVT::i8, MMO); SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewLD); SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) }; return DAG.getMergeValues(Ops, dl); } SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { if (Op.getOperand(1).getValueType().isVector()) return LowerVectorStore(Op, DAG); assert(Op.getOperand(1).getValueType() == MVT::i1 && "Custom lowering only for i1 stores"); // First, zero extend to 32 bits, then use a truncating store to 8 bits. SDLoc dl(Op); StoreSDNode *ST = cast(Op); SDValue Chain = ST->getChain(); SDValue BasePtr = ST->getBasePtr(); SDValue Value = ST->getValue(); MachineMemOperand *MMO = ST->getMemOperand(); Value = DAG.getNode(ISD::ZERO_EXTEND, dl, getPointerTy(DAG.getDataLayout()), Value); return DAG.getTruncStore(Chain, dl, Value, BasePtr, MVT::i8, MMO); } // FIXME: Remove this once the ANDI glue bug is fixed: SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const { assert(Op.getValueType() == MVT::i1 && "Custom lowering only for i1 results"); SDLoc DL(Op); return DAG.getNode(PPCISD::ANDIo_1_GT_BIT, DL, MVT::i1, Op.getOperand(0)); } SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op, SelectionDAG &DAG) const { // Implements a vector truncate that fits in a vector register as a shuffle. // We want to legalize vector truncates down to where the source fits in // a vector register (and target is therefore smaller than vector register // size). At that point legalization will try to custom lower the sub-legal // result and get here - where we can contain the truncate as a single target // operation. // For example a trunc <2 x i16> to <2 x i8> could be visualized as follows: // to // // We will implement it for big-endian ordering as this (where x denotes // undefined): // < MSB1|LSB1, MSB2|LSB2, uu, uu, uu, uu, uu, uu> to // < LSB1, LSB2, u, u, u, u, u, u, u, u, u, u, u, u, u, u> // // The same operation in little-endian ordering will be: // to // assert(Op.getValueType().isVector() && "Vector type expected."); SDLoc DL(Op); SDValue N1 = Op.getOperand(0); unsigned SrcSize = N1.getValueType().getSizeInBits(); assert(SrcSize <= 128 && "Source must fit in an Altivec/VSX vector"); SDValue WideSrc = SrcSize == 128 ? N1 : widenVec(DAG, N1, DL); EVT TrgVT = Op.getValueType(); unsigned TrgNumElts = TrgVT.getVectorNumElements(); EVT EltVT = TrgVT.getVectorElementType(); unsigned WideNumElts = 128 / EltVT.getSizeInBits(); EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideNumElts); // First list the elements we want to keep. unsigned SizeMult = SrcSize / TrgVT.getSizeInBits(); SmallVector ShuffV; if (Subtarget.isLittleEndian()) for (unsigned i = 0; i < TrgNumElts; ++i) ShuffV.push_back(i * SizeMult); else for (unsigned i = 1; i <= TrgNumElts; ++i) ShuffV.push_back(i * SizeMult - 1); // Populate the remaining elements with undefs. for (unsigned i = TrgNumElts; i < WideNumElts; ++i) // ShuffV.push_back(i + WideNumElts); ShuffV.push_back(WideNumElts + 1); SDValue Conv = DAG.getNode(ISD::BITCAST, DL, WideVT, WideSrc); return DAG.getVectorShuffle(WideVT, DL, Conv, DAG.getUNDEF(WideVT), ShuffV); } /// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when /// possible. SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { // Not FP? Not a fsel. if (!Op.getOperand(0).getValueType().isFloatingPoint() || !Op.getOperand(2).getValueType().isFloatingPoint()) return Op; // We might be able to do better than this under some circumstances, but in // general, fsel-based lowering of select is a finite-math-only optimization. // For more information, see section F.3 of the 2.06 ISA specification. if (!DAG.getTarget().Options.NoInfsFPMath || !DAG.getTarget().Options.NoNaNsFPMath) return Op; // TODO: Propagate flags from the select rather than global settings. SDNodeFlags Flags; Flags.setNoInfs(true); Flags.setNoNaNs(true); ISD::CondCode CC = cast(Op.getOperand(4))->get(); EVT ResVT = Op.getValueType(); EVT CmpVT = Op.getOperand(0).getValueType(); SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); SDValue TV = Op.getOperand(2), FV = Op.getOperand(3); SDLoc dl(Op); // If the RHS of the comparison is a 0.0, we don't need to do the // subtraction at all. SDValue Sel1; if (isFloatingPointZero(RHS)) switch (CC) { default: break; // SETUO etc aren't handled by fsel. case ISD::SETNE: std::swap(TV, FV); LLVM_FALLTHROUGH; case ISD::SETEQ: if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV); if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1); return DAG.getNode(PPCISD::FSEL, dl, ResVT, DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), Sel1, FV); case ISD::SETULT: case ISD::SETLT: std::swap(TV, FV); // fsel is natively setge, swap operands for setlt LLVM_FALLTHROUGH; case ISD::SETOGE: case ISD::SETGE: if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); return DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV); case ISD::SETUGT: case ISD::SETGT: std::swap(TV, FV); // fsel is natively setge, swap operands for setlt LLVM_FALLTHROUGH; case ISD::SETOLE: case ISD::SETLE: if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); return DAG.getNode(PPCISD::FSEL, dl, ResVT, DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), TV, FV); } SDValue Cmp; switch (CC) { default: break; // SETUO etc aren't handled by fsel. case ISD::SETNE: std::swap(TV, FV); LLVM_FALLTHROUGH; case ISD::SETEQ: Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1); return DAG.getNode(PPCISD::FSEL, dl, ResVT, DAG.getNode(ISD::FNEG, dl, MVT::f64, Cmp), Sel1, FV); case ISD::SETULT: case ISD::SETLT: Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV); case ISD::SETOGE: case ISD::SETGE: Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); case ISD::SETUGT: case ISD::SETGT: Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, Flags); if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV); case ISD::SETOLE: case ISD::SETLE: Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, Flags); if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); } return Op; } void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI, SelectionDAG &DAG, const SDLoc &dl) const { assert(Op.getOperand(0).getValueType().isFloatingPoint()); SDValue Src = Op.getOperand(0); if (Src.getValueType() == MVT::f32) Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); SDValue Tmp; switch (Op.getSimpleValueType().SimpleTy) { default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!"); case MVT::i32: Tmp = DAG.getNode( Op.getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIWZ : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ), dl, MVT::f64, Src); break; case MVT::i64: assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) && "i64 FP_TO_UINT is supported only with FPCVT"); Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ : PPCISD::FCTIDUZ, dl, MVT::f64, Src); break; } // Convert the FP value to an int value through memory. bool i32Stack = Op.getValueType() == MVT::i32 && Subtarget.hasSTFIWX() && (Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()); SDValue FIPtr = DAG.CreateStackTemporary(i32Stack ? MVT::i32 : MVT::f64); int FI = cast(FIPtr)->getIndex(); MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); // Emit a store to the stack slot. SDValue Chain; if (i32Stack) { MachineFunction &MF = DAG.getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, 4); SDValue Ops[] = { DAG.getEntryNode(), Tmp, FIPtr }; Chain = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl, DAG.getVTList(MVT::Other), Ops, MVT::i32, MMO); } else Chain = DAG.getStore(DAG.getEntryNode(), dl, Tmp, FIPtr, MPI); // Result is a load from the stack slot. If loading 4 bytes, make sure to // add in a bias on big endian. if (Op.getValueType() == MVT::i32 && !i32Stack) { FIPtr = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, DAG.getConstant(4, dl, FIPtr.getValueType())); MPI = MPI.getWithOffset(Subtarget.isLittleEndian() ? 0 : 4); } RLI.Chain = Chain; RLI.Ptr = FIPtr; RLI.MPI = MPI; } /// Custom lowers floating point to integer conversions to use /// the direct move instructions available in ISA 2.07 to avoid the /// need for load/store combinations. SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) const { assert(Op.getOperand(0).getValueType().isFloatingPoint()); SDValue Src = Op.getOperand(0); if (Src.getValueType() == MVT::f32) Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); SDValue Tmp; switch (Op.getSimpleValueType().SimpleTy) { default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!"); case MVT::i32: Tmp = DAG.getNode( Op.getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIWZ : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ), dl, MVT::f64, Src); Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i32, Tmp); break; case MVT::i64: assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) && "i64 FP_TO_UINT is supported only with FPCVT"); Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ : PPCISD::FCTIDUZ, dl, MVT::f64, Src); Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i64, Tmp); break; } return Tmp; } SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) const { // FP to INT conversions are legal for f128. if (EnableQuadPrecision && (Op->getOperand(0).getValueType() == MVT::f128)) return Op; // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on // PPC (the libcall is not available). if (Op.getOperand(0).getValueType() == MVT::ppcf128) { if (Op.getValueType() == MVT::i32) { if (Op.getOpcode() == ISD::FP_TO_SINT) { SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::f64, Op.getOperand(0), DAG.getIntPtrConstant(0, dl)); SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::f64, Op.getOperand(0), DAG.getIntPtrConstant(1, dl)); // Add the two halves of the long double in round-to-zero mode. SDValue Res = DAG.getNode(PPCISD::FADDRTZ, dl, MVT::f64, Lo, Hi); // Now use a smaller FP_TO_SINT. return DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Res); } if (Op.getOpcode() == ISD::FP_TO_UINT) { const uint64_t TwoE31[] = {0x41e0000000000000LL, 0}; APFloat APF = APFloat(APFloat::PPCDoubleDouble(), APInt(128, TwoE31)); SDValue Tmp = DAG.getConstantFP(APF, dl, MVT::ppcf128); // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X // FIXME: generated code sucks. // TODO: Are there fast-math-flags to propagate to this FSUB? SDValue True = DAG.getNode(ISD::FSUB, dl, MVT::ppcf128, Op.getOperand(0), Tmp); True = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, True); True = DAG.getNode(ISD::ADD, dl, MVT::i32, True, DAG.getConstant(0x80000000, dl, MVT::i32)); SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op.getOperand(0)); return DAG.getSelectCC(dl, Op.getOperand(0), Tmp, True, False, ISD::SETGE); } } return SDValue(); } if (Subtarget.hasDirectMove() && Subtarget.isPPC64()) return LowerFP_TO_INTDirectMove(Op, DAG, dl); ReuseLoadInfo RLI; LowerFP_TO_INTForReuse(Op, RLI, DAG, dl); return DAG.getLoad(Op.getValueType(), dl, RLI.Chain, RLI.Ptr, RLI.MPI, RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges); } // We're trying to insert a regular store, S, and then a load, L. If the // incoming value, O, is a load, we might just be able to have our load use the // address used by O. However, we don't know if anything else will store to // that address before we can load from it. To prevent this situation, we need // to insert our load, L, into the chain as a peer of O. To do this, we give L // the same chain operand as O, we create a token factor from the chain results // of O and L, and we replace all uses of O's chain result with that token // factor (see spliceIntoChain below for this last part). bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT, ReuseLoadInfo &RLI, SelectionDAG &DAG, ISD::LoadExtType ET) const { SDLoc dl(Op); if (ET == ISD::NON_EXTLOAD && (Op.getOpcode() == ISD::FP_TO_UINT || Op.getOpcode() == ISD::FP_TO_SINT) && isOperationLegalOrCustom(Op.getOpcode(), Op.getOperand(0).getValueType())) { LowerFP_TO_INTForReuse(Op, RLI, DAG, dl); return true; } LoadSDNode *LD = dyn_cast(Op); if (!LD || LD->getExtensionType() != ET || LD->isVolatile() || LD->isNonTemporal()) return false; if (LD->getMemoryVT() != MemVT) return false; RLI.Ptr = LD->getBasePtr(); if (LD->isIndexed() && !LD->getOffset().isUndef()) { assert(LD->getAddressingMode() == ISD::PRE_INC && "Non-pre-inc AM on PPC?"); RLI.Ptr = DAG.getNode(ISD::ADD, dl, RLI.Ptr.getValueType(), RLI.Ptr, LD->getOffset()); } RLI.Chain = LD->getChain(); RLI.MPI = LD->getPointerInfo(); RLI.IsDereferenceable = LD->isDereferenceable(); RLI.IsInvariant = LD->isInvariant(); RLI.Alignment = LD->getAlignment(); RLI.AAInfo = LD->getAAInfo(); RLI.Ranges = LD->getRanges(); RLI.ResChain = SDValue(LD, LD->isIndexed() ? 2 : 1); return true; } // Given the head of the old chain, ResChain, insert a token factor containing // it and NewResChain, and make users of ResChain now be users of that token // factor. // TODO: Remove and use DAG::makeEquivalentMemoryOrdering() instead. void PPCTargetLowering::spliceIntoChain(SDValue ResChain, SDValue NewResChain, SelectionDAG &DAG) const { if (!ResChain) return; SDLoc dl(NewResChain); SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, NewResChain, DAG.getUNDEF(MVT::Other)); assert(TF.getNode() != NewResChain.getNode() && "A new TF really is required here"); DAG.ReplaceAllUsesOfValueWith(ResChain, TF); DAG.UpdateNodeOperands(TF.getNode(), ResChain, NewResChain); } /// Analyze profitability of direct move /// prefer float load to int load plus direct move /// when there is no integer use of int load bool PPCTargetLowering::directMoveIsProfitable(const SDValue &Op) const { SDNode *Origin = Op.getOperand(0).getNode(); if (Origin->getOpcode() != ISD::LOAD) return true; // If there is no LXSIBZX/LXSIHZX, like Power8, // prefer direct move if the memory size is 1 or 2 bytes. MachineMemOperand *MMO = cast(Origin)->getMemOperand(); if (!Subtarget.hasP9Vector() && MMO->getSize() <= 2) return true; for (SDNode::use_iterator UI = Origin->use_begin(), UE = Origin->use_end(); UI != UE; ++UI) { // Only look at the users of the loaded value. if (UI.getUse().get().getResNo() != 0) continue; if (UI->getOpcode() != ISD::SINT_TO_FP && UI->getOpcode() != ISD::UINT_TO_FP) return true; } return false; } /// Custom lowers integer to floating point conversions to use /// the direct move instructions available in ISA 2.07 to avoid the /// need for load/store combinations. SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) const { assert((Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::f64) && "Invalid floating point type as target of conversion"); assert(Subtarget.hasFPCVT() && "Int to FP conversions with direct moves require FPCVT"); SDValue FP; SDValue Src = Op.getOperand(0); bool SinglePrec = Op.getValueType() == MVT::f32; bool WordInt = Src.getSimpleValueType().SimpleTy == MVT::i32; bool Signed = Op.getOpcode() == ISD::SINT_TO_FP; unsigned ConvOp = Signed ? (SinglePrec ? PPCISD::FCFIDS : PPCISD::FCFID) : (SinglePrec ? PPCISD::FCFIDUS : PPCISD::FCFIDU); if (WordInt) { FP = DAG.getNode(Signed ? PPCISD::MTVSRA : PPCISD::MTVSRZ, dl, MVT::f64, Src); FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP); } else { FP = DAG.getNode(PPCISD::MTVSRA, dl, MVT::f64, Src); FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP); } return FP; } static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) { EVT VecVT = Vec.getValueType(); assert(VecVT.isVector() && "Expected a vector type."); assert(VecVT.getSizeInBits() < 128 && "Vector is already full width."); EVT EltVT = VecVT.getVectorElementType(); unsigned WideNumElts = 128 / EltVT.getSizeInBits(); EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideNumElts); unsigned NumConcat = WideNumElts / VecVT.getVectorNumElements(); SmallVector Ops(NumConcat); Ops[0] = Vec; SDValue UndefVec = DAG.getUNDEF(VecVT); for (unsigned i = 1; i < NumConcat; ++i) Ops[i] = UndefVec; return DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Ops); } SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) const { unsigned Opc = Op.getOpcode(); assert((Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP) && "Unexpected conversion type"); assert((Op.getValueType() == MVT::v2f64 || Op.getValueType() == MVT::v4f32) && "Supports conversions to v2f64/v4f32 only."); bool SignedConv = Opc == ISD::SINT_TO_FP; bool FourEltRes = Op.getValueType() == MVT::v4f32; SDValue Wide = widenVec(DAG, Op.getOperand(0), dl); EVT WideVT = Wide.getValueType(); unsigned WideNumElts = WideVT.getVectorNumElements(); MVT IntermediateVT = FourEltRes ? MVT::v4i32 : MVT::v2i64; SmallVector ShuffV; for (unsigned i = 0; i < WideNumElts; ++i) ShuffV.push_back(i + WideNumElts); int Stride = FourEltRes ? WideNumElts / 4 : WideNumElts / 2; int SaveElts = FourEltRes ? 4 : 2; if (Subtarget.isLittleEndian()) for (int i = 0; i < SaveElts; i++) ShuffV[i * Stride] = i; else for (int i = 1; i <= SaveElts; i++) ShuffV[i * Stride - 1] = i - 1; SDValue ShuffleSrc2 = SignedConv ? DAG.getUNDEF(WideVT) : DAG.getConstant(0, dl, WideVT); SDValue Arrange = DAG.getVectorShuffle(WideVT, dl, Wide, ShuffleSrc2, ShuffV); unsigned ExtendOp = SignedConv ? (unsigned)PPCISD::SExtVElems : (unsigned)ISD::BITCAST; SDValue Extend; if (!Subtarget.hasP9Altivec() && SignedConv) { Arrange = DAG.getBitcast(IntermediateVT, Arrange); Extend = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, IntermediateVT, Arrange, DAG.getValueType(Op.getOperand(0).getValueType())); } else Extend = DAG.getNode(ExtendOp, dl, IntermediateVT, Arrange); return DAG.getNode(Opc, dl, Op.getValueType(), Extend); } SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); EVT InVT = Op.getOperand(0).getValueType(); EVT OutVT = Op.getValueType(); if (OutVT.isVector() && OutVT.isFloatingPoint() && isOperationCustom(Op.getOpcode(), InVT)) return LowerINT_TO_FPVector(Op, DAG, dl); // Conversions to f128 are legal. if (EnableQuadPrecision && (Op.getValueType() == MVT::f128)) return Op; if (Subtarget.hasQPX() && Op.getOperand(0).getValueType() == MVT::v4i1) { if (Op.getValueType() != MVT::v4f32 && Op.getValueType() != MVT::v4f64) return SDValue(); SDValue Value = Op.getOperand(0); // The values are now known to be -1 (false) or 1 (true). To convert this // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64); Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); if (Op.getValueType() != MVT::v4f64) Value = DAG.getNode(ISD::FP_ROUND, dl, Op.getValueType(), Value, DAG.getIntPtrConstant(1, dl)); return Value; } // Don't handle ppc_fp128 here; let it be lowered to a libcall. if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64) return SDValue(); if (Op.getOperand(0).getValueType() == MVT::i1) return DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Op.getOperand(0), DAG.getConstantFP(1.0, dl, Op.getValueType()), DAG.getConstantFP(0.0, dl, Op.getValueType())); // If we have direct moves, we can do all the conversion, skip the store/load // however, without FPCVT we can't do most conversions. if (Subtarget.hasDirectMove() && directMoveIsProfitable(Op) && Subtarget.isPPC64() && Subtarget.hasFPCVT()) return LowerINT_TO_FPDirectMove(Op, DAG, dl); assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) && "UINT_TO_FP is supported only with FPCVT"); // If we have FCFIDS, then use it when converting to single-precision. // Otherwise, convert to double-precision and then round. unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS : PPCISD::FCFIDS) : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU : PPCISD::FCFID); MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) ? MVT::f32 : MVT::f64; if (Op.getOperand(0).getValueType() == MVT::i64) { SDValue SINT = Op.getOperand(0); // When converting to single-precision, we actually need to convert // to double-precision first and then round to single-precision. // To avoid double-rounding effects during that operation, we have // to prepare the input operand. Bits that might be truncated when // converting to double-precision are replaced by a bit that won't // be lost at this stage, but is below the single-precision rounding // position. // // However, if -enable-unsafe-fp-math is in effect, accept double // rounding to avoid the extra overhead. if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT() && !DAG.getTarget().Options.UnsafeFPMath) { // Twiddle input to make sure the low 11 bits are zero. (If this // is the case, we are guaranteed the value will fit into the 53 bit // mantissa of an IEEE double-precision value without rounding.) // If any of those low 11 bits were not zero originally, make sure // bit 12 (value 2048) is set instead, so that the final rounding // to single-precision gets the correct result. SDValue Round = DAG.getNode(ISD::AND, dl, MVT::i64, SINT, DAG.getConstant(2047, dl, MVT::i64)); Round = DAG.getNode(ISD::ADD, dl, MVT::i64, Round, DAG.getConstant(2047, dl, MVT::i64)); Round = DAG.getNode(ISD::OR, dl, MVT::i64, Round, SINT); Round = DAG.getNode(ISD::AND, dl, MVT::i64, Round, DAG.getConstant(-2048, dl, MVT::i64)); // However, we cannot use that value unconditionally: if the magnitude // of the input value is small, the bit-twiddling we did above might // end up visibly changing the output. Fortunately, in that case, we // don't need to twiddle bits since the original input will convert // exactly to double-precision floating-point already. Therefore, // construct a conditional to use the original value if the top 11 // bits are all sign-bit copies, and use the rounded value computed // above otherwise. SDValue Cond = DAG.getNode(ISD::SRA, dl, MVT::i64, SINT, DAG.getConstant(53, dl, MVT::i32)); Cond = DAG.getNode(ISD::ADD, dl, MVT::i64, Cond, DAG.getConstant(1, dl, MVT::i64)); Cond = DAG.getSetCC(dl, MVT::i32, Cond, DAG.getConstant(1, dl, MVT::i64), ISD::SETUGT); SINT = DAG.getNode(ISD::SELECT, dl, MVT::i64, Cond, Round, SINT); } ReuseLoadInfo RLI; SDValue Bits; MachineFunction &MF = DAG.getMachineFunction(); if (canReuseLoadAddress(SINT, MVT::i64, RLI, DAG)) { Bits = DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI, RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges); spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); } else if (Subtarget.hasLFIWAX() && canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::SEXTLOAD)) { MachineMemOperand *MMO = MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment, RLI.AAInfo, RLI.Ranges); SDValue Ops[] = { RLI.Chain, RLI.Ptr }; Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWAX, dl, DAG.getVTList(MVT::f64, MVT::Other), Ops, MVT::i32, MMO); spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); } else if (Subtarget.hasFPCVT() && canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::ZEXTLOAD)) { MachineMemOperand *MMO = MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment, RLI.AAInfo, RLI.Ranges); SDValue Ops[] = { RLI.Chain, RLI.Ptr }; Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWZX, dl, DAG.getVTList(MVT::f64, MVT::Other), Ops, MVT::i32, MMO); spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); } else if (((Subtarget.hasLFIWAX() && SINT.getOpcode() == ISD::SIGN_EXTEND) || (Subtarget.hasFPCVT() && SINT.getOpcode() == ISD::ZERO_EXTEND)) && SINT.getOperand(0).getValueType() == MVT::i32) { MachineFrameInfo &MFI = MF.getFrameInfo(); EVT PtrVT = getPointerTy(DAG.getDataLayout()); int FrameIdx = MFI.CreateStackObject(4, 4, false); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, SINT.getOperand(0), FIdx, MachinePointerInfo::getFixedStack( DAG.getMachineFunction(), FrameIdx)); assert(cast(Store)->getMemoryVT() == MVT::i32 && "Expected an i32 store"); RLI.Ptr = FIdx; RLI.Chain = Store; RLI.MPI = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); RLI.Alignment = 4; MachineMemOperand *MMO = MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment, RLI.AAInfo, RLI.Ranges); SDValue Ops[] = { RLI.Chain, RLI.Ptr }; Bits = DAG.getMemIntrinsicNode(SINT.getOpcode() == ISD::ZERO_EXTEND ? PPCISD::LFIWZX : PPCISD::LFIWAX, dl, DAG.getVTList(MVT::f64, MVT::Other), Ops, MVT::i32, MMO); } else Bits = DAG.getNode(ISD::BITCAST, dl, MVT::f64, SINT); SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Bits); if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, DAG.getIntPtrConstant(0, dl)); return FP; } assert(Op.getOperand(0).getValueType() == MVT::i32 && "Unhandled INT_TO_FP type in custom expander!"); // Since we only generate this in 64-bit mode, we can take advantage of // 64-bit registers. In particular, sign extend the input value into the // 64-bit register with extsw, store the WHOLE 64-bit value into the stack // then lfd it and fcfid it. MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); EVT PtrVT = getPointerTy(MF.getDataLayout()); SDValue Ld; if (Subtarget.hasLFIWAX() || Subtarget.hasFPCVT()) { ReuseLoadInfo RLI; bool ReusingLoad; if (!(ReusingLoad = canReuseLoadAddress(Op.getOperand(0), MVT::i32, RLI, DAG))) { int FrameIdx = MFI.CreateStackObject(4, 4, false); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx, MachinePointerInfo::getFixedStack( DAG.getMachineFunction(), FrameIdx)); assert(cast(Store)->getMemoryVT() == MVT::i32 && "Expected an i32 store"); RLI.Ptr = FIdx; RLI.Chain = Store; RLI.MPI = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); RLI.Alignment = 4; } MachineMemOperand *MMO = MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment, RLI.AAInfo, RLI.Ranges); SDValue Ops[] = { RLI.Chain, RLI.Ptr }; Ld = DAG.getMemIntrinsicNode(Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::LFIWZX : PPCISD::LFIWAX, dl, DAG.getVTList(MVT::f64, MVT::Other), Ops, MVT::i32, MMO); if (ReusingLoad) spliceIntoChain(RLI.ResChain, Ld.getValue(1), DAG); } else { assert(Subtarget.isPPC64() && "i32->FP without LFIWAX supported only on PPC64"); int FrameIdx = MFI.CreateStackObject(8, 8, false); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); SDValue Ext64 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i64, Op.getOperand(0)); // STD the extended value into the stack slot. SDValue Store = DAG.getStore( DAG.getEntryNode(), dl, Ext64, FIdx, MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx)); // Load the value as a double. Ld = DAG.getLoad( MVT::f64, dl, Store, FIdx, MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx)); } // FCFID it and return it. SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Ld); if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, DAG.getIntPtrConstant(0, dl)); return FP; } SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); /* The rounding mode is in bits 30:31 of FPSR, and has the following settings: 00 Round to nearest 01 Round to 0 10 Round to +inf 11 Round to -inf FLT_ROUNDS, on the other hand, expects the following: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to +inf 3 Round to -inf To perform the conversion, we do: ((FPSCR & 0x3) ^ ((~FPSCR & 0x3) >> 1)) */ MachineFunction &MF = DAG.getMachineFunction(); EVT VT = Op.getValueType(); EVT PtrVT = getPointerTy(MF.getDataLayout()); // Save FP Control Word to register EVT NodeTys[] = { MVT::f64, // return register MVT::Glue // unused in this context }; SDValue Chain = DAG.getNode(PPCISD::MFFS, dl, NodeTys, None); // Save FP register to stack slot int SSFI = MF.getFrameInfo().CreateStackObject(8, 8, false); SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT); SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Chain, StackSlot, MachinePointerInfo()); // Load FP Control Word from low 32 bits of stack slot. SDValue Four = DAG.getConstant(4, dl, PtrVT); SDValue Addr = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, Four); SDValue CWD = DAG.getLoad(MVT::i32, dl, Store, Addr, MachinePointerInfo()); // Transform as necessary SDValue CWD1 = DAG.getNode(ISD::AND, dl, MVT::i32, CWD, DAG.getConstant(3, dl, MVT::i32)); SDValue CWD2 = DAG.getNode(ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::AND, dl, MVT::i32, DAG.getNode(ISD::XOR, dl, MVT::i32, CWD, DAG.getConstant(3, dl, MVT::i32)), DAG.getConstant(3, dl, MVT::i32)), DAG.getConstant(1, dl, MVT::i32)); SDValue RetVal = DAG.getNode(ISD::XOR, dl, MVT::i32, CWD1, CWD2); return DAG.getNode((VT.getSizeInBits() < 16 ? ISD::TRUNCATE : ISD::ZERO_EXTEND), dl, VT, RetVal); } SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const { EVT VT = Op.getValueType(); unsigned BitWidth = VT.getSizeInBits(); SDLoc dl(Op); assert(Op.getNumOperands() == 3 && VT == Op.getOperand(1).getValueType() && "Unexpected SHL!"); // Expand into a bunch of logical ops. Note that these ops // depend on the PPC behavior for oversized shift amounts. SDValue Lo = Op.getOperand(0); SDValue Hi = Op.getOperand(1); SDValue Amt = Op.getOperand(2); EVT AmtVT = Amt.getValueType(); SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, DAG.getConstant(BitWidth, dl, AmtVT), Amt); SDValue Tmp2 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Amt); SDValue Tmp3 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Tmp1); SDValue Tmp4 = DAG.getNode(ISD::OR , dl, VT, Tmp2, Tmp3); SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, DAG.getConstant(-BitWidth, dl, AmtVT)); SDValue Tmp6 = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Tmp5); SDValue OutHi = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6); SDValue OutLo = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Amt); SDValue OutOps[] = { OutLo, OutHi }; return DAG.getMergeValues(OutOps, dl); } SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const { EVT VT = Op.getValueType(); SDLoc dl(Op); unsigned BitWidth = VT.getSizeInBits(); assert(Op.getNumOperands() == 3 && VT == Op.getOperand(1).getValueType() && "Unexpected SRL!"); // Expand into a bunch of logical ops. Note that these ops // depend on the PPC behavior for oversized shift amounts. SDValue Lo = Op.getOperand(0); SDValue Hi = Op.getOperand(1); SDValue Amt = Op.getOperand(2); EVT AmtVT = Amt.getValueType(); SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, DAG.getConstant(BitWidth, dl, AmtVT), Amt); SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt); SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1); SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, DAG.getConstant(-BitWidth, dl, AmtVT)); SDValue Tmp6 = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Tmp5); SDValue OutLo = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6); SDValue OutHi = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Amt); SDValue OutOps[] = { OutLo, OutHi }; return DAG.getMergeValues(OutOps, dl); } SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); EVT VT = Op.getValueType(); unsigned BitWidth = VT.getSizeInBits(); assert(Op.getNumOperands() == 3 && VT == Op.getOperand(1).getValueType() && "Unexpected SRA!"); // Expand into a bunch of logical ops, followed by a select_cc. SDValue Lo = Op.getOperand(0); SDValue Hi = Op.getOperand(1); SDValue Amt = Op.getOperand(2); EVT AmtVT = Amt.getValueType(); SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, DAG.getConstant(BitWidth, dl, AmtVT), Amt); SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt); SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1); SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, DAG.getConstant(-BitWidth, dl, AmtVT)); SDValue Tmp6 = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Tmp5); SDValue OutHi = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Amt); SDValue OutLo = DAG.getSelectCC(dl, Tmp5, DAG.getConstant(0, dl, AmtVT), Tmp4, Tmp6, ISD::SETLE); SDValue OutOps[] = { OutLo, OutHi }; return DAG.getMergeValues(OutOps, dl); } //===----------------------------------------------------------------------===// // Vector related lowering. // /// BuildSplatI - Build a canonical splati of Val with an element size of /// SplatSize. Cast the result to VT. static SDValue BuildSplatI(int Val, unsigned SplatSize, EVT VT, SelectionDAG &DAG, const SDLoc &dl) { assert(Val >= -16 && Val <= 15 && "vsplti is out of range!"); static const MVT VTys[] = { // canonical VT to use for each size. MVT::v16i8, MVT::v8i16, MVT::Other, MVT::v4i32 }; EVT ReqVT = VT != MVT::Other ? VT : VTys[SplatSize-1]; // Force vspltis[hw] -1 to vspltisb -1 to canonicalize. if (Val == -1) SplatSize = 1; EVT CanonicalVT = VTys[SplatSize-1]; // Build a canonical splat for this value. return DAG.getBitcast(ReqVT, DAG.getConstant(Val, dl, CanonicalVT)); } /// BuildIntrinsicOp - Return a unary operator intrinsic node with the /// specified intrinsic ID. static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op, SelectionDAG &DAG, const SDLoc &dl, EVT DestVT = MVT::Other) { if (DestVT == MVT::Other) DestVT = Op.getValueType(); return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, DAG.getConstant(IID, dl, MVT::i32), Op); } /// BuildIntrinsicOp - Return a binary operator intrinsic node with the /// specified intrinsic ID. static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl, EVT DestVT = MVT::Other) { if (DestVT == MVT::Other) DestVT = LHS.getValueType(); return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, DAG.getConstant(IID, dl, MVT::i32), LHS, RHS); } /// BuildIntrinsicOp - Return a ternary operator intrinsic node with the /// specified intrinsic ID. static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1, SDValue Op2, SelectionDAG &DAG, const SDLoc &dl, EVT DestVT = MVT::Other) { if (DestVT == MVT::Other) DestVT = Op0.getValueType(); return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, DAG.getConstant(IID, dl, MVT::i32), Op0, Op1, Op2); } /// BuildVSLDOI - Return a VECTOR_SHUFFLE that is a vsldoi of the specified /// amount. The result has the specified value type. static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt, EVT VT, SelectionDAG &DAG, const SDLoc &dl) { // Force LHS/RHS to be the right type. LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, LHS); RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, RHS); int Ops[16]; for (unsigned i = 0; i != 16; ++i) Ops[i] = i + Amt; SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, LHS, RHS, Ops); return DAG.getNode(ISD::BITCAST, dl, VT, T); } /// Do we have an efficient pattern in a .td file for this node? /// /// \param V - pointer to the BuildVectorSDNode being matched /// \param HasDirectMove - does this subtarget have VSR <-> GPR direct moves? /// /// There are some patterns where it is beneficial to keep a BUILD_VECTOR /// node as a BUILD_VECTOR node rather than expanding it. The patterns where /// the opposite is true (expansion is beneficial) are: /// - The node builds a vector out of integers that are not 32 or 64-bits /// - The node builds a vector out of constants /// - The node is a "load-and-splat" /// In all other cases, we will choose to keep the BUILD_VECTOR. static bool haveEfficientBuildVectorPattern(BuildVectorSDNode *V, bool HasDirectMove, bool HasP8Vector) { EVT VecVT = V->getValueType(0); bool RightType = VecVT == MVT::v2f64 || (HasP8Vector && VecVT == MVT::v4f32) || (HasDirectMove && (VecVT == MVT::v2i64 || VecVT == MVT::v4i32)); if (!RightType) return false; bool IsSplat = true; bool IsLoad = false; SDValue Op0 = V->getOperand(0); // This function is called in a block that confirms the node is not a constant // splat. So a constant BUILD_VECTOR here means the vector is built out of // different constants. if (V->isConstant()) return false; for (int i = 0, e = V->getNumOperands(); i < e; ++i) { if (V->getOperand(i).isUndef()) return false; // We want to expand nodes that represent load-and-splat even if the // loaded value is a floating point truncation or conversion to int. if (V->getOperand(i).getOpcode() == ISD::LOAD || (V->getOperand(i).getOpcode() == ISD::FP_ROUND && V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD) || (V->getOperand(i).getOpcode() == ISD::FP_TO_SINT && V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD) || (V->getOperand(i).getOpcode() == ISD::FP_TO_UINT && V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD)) IsLoad = true; // If the operands are different or the input is not a load and has more // uses than just this BV node, then it isn't a splat. if (V->getOperand(i) != Op0 || (!IsLoad && !V->isOnlyUserOf(V->getOperand(i).getNode()))) IsSplat = false; } return !(IsSplat && IsLoad); } // Lower BITCAST(f128, (build_pair i64, i64)) to BUILD_FP128. SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); SDValue Op0 = Op->getOperand(0); if (!EnableQuadPrecision || (Op.getValueType() != MVT::f128 ) || (Op0.getOpcode() != ISD::BUILD_PAIR) || (Op0.getOperand(0).getValueType() != MVT::i64) || (Op0.getOperand(1).getValueType() != MVT::i64)) return SDValue(); return DAG.getNode(PPCISD::BUILD_FP128, dl, MVT::f128, Op0.getOperand(0), Op0.getOperand(1)); } // If this is a case we can't handle, return null and let the default // expansion code take care of it. If we CAN select this case, and if it // selects to a single instruction, return Op. Otherwise, if we can codegen // this case more efficiently than a constant pool load, lower it to the // sequence of ops that should be used. SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); BuildVectorSDNode *BVN = dyn_cast(Op.getNode()); assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR"); if (Subtarget.hasQPX() && Op.getValueType() == MVT::v4i1) { // We first build an i32 vector, load it into a QPX register, // then convert it to a floating-point vector and compare it // to a zero vector to get the boolean result. MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); int FrameIdx = MFI.CreateStackObject(16, 16, false); MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); EVT PtrVT = getPointerTy(DAG.getDataLayout()); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); assert(BVN->getNumOperands() == 4 && "BUILD_VECTOR for v4i1 does not have 4 operands"); bool IsConst = true; for (unsigned i = 0; i < 4; ++i) { if (BVN->getOperand(i).isUndef()) continue; if (!isa(BVN->getOperand(i))) { IsConst = false; break; } } if (IsConst) { Constant *One = ConstantFP::get(Type::getFloatTy(*DAG.getContext()), 1.0); Constant *NegOne = ConstantFP::get(Type::getFloatTy(*DAG.getContext()), -1.0); Constant *CV[4]; for (unsigned i = 0; i < 4; ++i) { if (BVN->getOperand(i).isUndef()) CV[i] = UndefValue::get(Type::getFloatTy(*DAG.getContext())); else if (isNullConstant(BVN->getOperand(i))) CV[i] = NegOne; else CV[i] = One; } Constant *CP = ConstantVector::get(CV); SDValue CPIdx = DAG.getConstantPool(CP, getPointerTy(DAG.getDataLayout()), 16 /* alignment */); SDValue Ops[] = {DAG.getEntryNode(), CPIdx}; SDVTList VTs = DAG.getVTList({MVT::v4i1, /*chain*/ MVT::Other}); return DAG.getMemIntrinsicNode( PPCISD::QVLFSb, dl, VTs, Ops, MVT::v4f32, MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); } SmallVector Stores; for (unsigned i = 0; i < 4; ++i) { if (BVN->getOperand(i).isUndef()) continue; unsigned Offset = 4*i; SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); unsigned StoreSize = BVN->getOperand(i).getValueType().getStoreSize(); if (StoreSize > 4) { Stores.push_back( DAG.getTruncStore(DAG.getEntryNode(), dl, BVN->getOperand(i), Idx, PtrInfo.getWithOffset(Offset), MVT::i32)); } else { SDValue StoreValue = BVN->getOperand(i); if (StoreSize < 4) StoreValue = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, StoreValue); Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, StoreValue, Idx, PtrInfo.getWithOffset(Offset))); } } SDValue StoreChain; if (!Stores.empty()) StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); else StoreChain = DAG.getEntryNode(); // Now load from v4i32 into the QPX register; this will extend it to // v4i64 but not yet convert it to a floating point. Nevertheless, this // is typed as v4f64 because the QPX register integer states are not // explicitly represented. SDValue Ops[] = {StoreChain, DAG.getConstant(Intrinsic::ppc_qpx_qvlfiwz, dl, MVT::i32), FIdx}; SDVTList VTs = DAG.getVTList({MVT::v4f64, /*chain*/ MVT::Other}); SDValue LoadedVect = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, VTs, Ops, MVT::v4i32, PtrInfo); LoadedVect = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, DAG.getConstant(Intrinsic::ppc_qpx_qvfcfidu, dl, MVT::i32), LoadedVect); SDValue FPZeros = DAG.getConstantFP(0.0, dl, MVT::v4f64); return DAG.getSetCC(dl, MVT::v4i1, LoadedVect, FPZeros, ISD::SETEQ); } // All other QPX vectors are handled by generic code. if (Subtarget.hasQPX()) return SDValue(); // Check if this is a splat of a constant value. APInt APSplatBits, APSplatUndef; unsigned SplatBitSize; bool HasAnyUndefs; if (! BVN->isConstantSplat(APSplatBits, APSplatUndef, SplatBitSize, HasAnyUndefs, 0, !Subtarget.isLittleEndian()) || SplatBitSize > 32) { // BUILD_VECTOR nodes that are not constant splats of up to 32-bits can be // lowered to VSX instructions under certain conditions. // Without VSX, there is no pattern more efficient than expanding the node. if (Subtarget.hasVSX() && haveEfficientBuildVectorPattern(BVN, Subtarget.hasDirectMove(), Subtarget.hasP8Vector())) return Op; return SDValue(); } unsigned SplatBits = APSplatBits.getZExtValue(); unsigned SplatUndef = APSplatUndef.getZExtValue(); unsigned SplatSize = SplatBitSize / 8; // First, handle single instruction cases. // All zeros? if (SplatBits == 0) { // Canonicalize all zero vectors to be v4i32. if (Op.getValueType() != MVT::v4i32 || HasAnyUndefs) { SDValue Z = DAG.getConstant(0, dl, MVT::v4i32); Op = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Z); } return Op; } // We have XXSPLTIB for constant splats one byte wide if (Subtarget.hasP9Vector() && SplatSize == 1) { // This is a splat of 1-byte elements with some elements potentially undef. // Rather than trying to match undef in the SDAG patterns, ensure that all // elements are the same constant. if (HasAnyUndefs || ISD::isBuildVectorAllOnes(BVN)) { SmallVector Ops(16, DAG.getConstant(SplatBits, dl, MVT::i32)); SDValue NewBV = DAG.getBuildVector(MVT::v16i8, dl, Ops); if (Op.getValueType() != MVT::v16i8) return DAG.getBitcast(Op.getValueType(), NewBV); return NewBV; } // BuildVectorSDNode::isConstantSplat() is actually pretty smart. It'll // detect that constant splats like v8i16: 0xABAB are really just splats // of a 1-byte constant. In this case, we need to convert the node to a // splat of v16i8 and a bitcast. if (Op.getValueType() != MVT::v16i8) return DAG.getBitcast(Op.getValueType(), DAG.getConstant(SplatBits, dl, MVT::v16i8)); return Op; } // If the sign extended value is in the range [-16,15], use VSPLTI[bhw]. int32_t SextVal= (int32_t(SplatBits << (32-SplatBitSize)) >> (32-SplatBitSize)); if (SextVal >= -16 && SextVal <= 15) return BuildSplatI(SextVal, SplatSize, Op.getValueType(), DAG, dl); // Two instruction sequences. // If this value is in the range [-32,30] and is even, use: // VSPLTI[bhw](val/2) + VSPLTI[bhw](val/2) // If this value is in the range [17,31] and is odd, use: // VSPLTI[bhw](val-16) - VSPLTI[bhw](-16) // If this value is in the range [-31,-17] and is odd, use: // VSPLTI[bhw](val+16) + VSPLTI[bhw](-16) // Note the last two are three-instruction sequences. if (SextVal >= -32 && SextVal <= 31) { // To avoid having these optimizations undone by constant folding, // we convert to a pseudo that will be expanded later into one of // the above forms. SDValue Elt = DAG.getConstant(SextVal, dl, MVT::i32); EVT VT = (SplatSize == 1 ? MVT::v16i8 : (SplatSize == 2 ? MVT::v8i16 : MVT::v4i32)); SDValue EltSize = DAG.getConstant(SplatSize, dl, MVT::i32); SDValue RetVal = DAG.getNode(PPCISD::VADD_SPLAT, dl, VT, Elt, EltSize); if (VT == Op.getValueType()) return RetVal; else return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), RetVal); } // If this is 0x8000_0000 x 4, turn into vspltisw + vslw. If it is // 0x7FFF_FFFF x 4, turn it into not(0x8000_0000). This is important // for fneg/fabs. if (SplatSize == 4 && SplatBits == (0x7FFFFFFF&~SplatUndef)) { // Make -1 and vspltisw -1: SDValue OnesV = BuildSplatI(-1, 4, MVT::v4i32, DAG, dl); // Make the VSLW intrinsic, computing 0x8000_0000. SDValue Res = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, OnesV, OnesV, DAG, dl); // xor by OnesV to invert it. Res = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Res, OnesV); return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); } // Check to see if this is a wide variety of vsplti*, binop self cases. static const signed char SplatCsts[] = { -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7, -8, 8, -9, 9, -10, 10, -11, 11, -12, 12, -13, 13, 14, -14, 15, -15, -16 }; for (unsigned idx = 0; idx < array_lengthof(SplatCsts); ++idx) { // Indirect through the SplatCsts array so that we favor 'vsplti -1' for // cases which are ambiguous (e.g. formation of 0x8000_0000). 'vsplti -1' int i = SplatCsts[idx]; // Figure out what shift amount will be used by altivec if shifted by i in // this splat size. unsigned TypeShiftAmt = i & (SplatBitSize-1); // vsplti + shl self. if (SextVal == (int)((unsigned)i << TypeShiftAmt)) { SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); static const unsigned IIDs[] = { // Intrinsic to use for each size. Intrinsic::ppc_altivec_vslb, Intrinsic::ppc_altivec_vslh, 0, Intrinsic::ppc_altivec_vslw }; Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); } // vsplti + srl self. if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) { SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); static const unsigned IIDs[] = { // Intrinsic to use for each size. Intrinsic::ppc_altivec_vsrb, Intrinsic::ppc_altivec_vsrh, 0, Intrinsic::ppc_altivec_vsrw }; Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); } // vsplti + sra self. if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) { SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); static const unsigned IIDs[] = { // Intrinsic to use for each size. Intrinsic::ppc_altivec_vsrab, Intrinsic::ppc_altivec_vsrah, 0, Intrinsic::ppc_altivec_vsraw }; Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); } // vsplti + rol self. if (SextVal == (int)(((unsigned)i << TypeShiftAmt) | ((unsigned)i >> (SplatBitSize-TypeShiftAmt)))) { SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); static const unsigned IIDs[] = { // Intrinsic to use for each size. Intrinsic::ppc_altivec_vrlb, Intrinsic::ppc_altivec_vrlh, 0, Intrinsic::ppc_altivec_vrlw }; Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); } // t = vsplti c, result = vsldoi t, t, 1 if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) { SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); unsigned Amt = Subtarget.isLittleEndian() ? 15 : 1; return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); } // t = vsplti c, result = vsldoi t, t, 2 if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) { SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); unsigned Amt = Subtarget.isLittleEndian() ? 14 : 2; return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); } // t = vsplti c, result = vsldoi t, t, 3 if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) { SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); unsigned Amt = Subtarget.isLittleEndian() ? 13 : 3; return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); } } return SDValue(); } /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit /// the specified operations to build the shuffle. static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl) { unsigned OpNum = (PFEntry >> 26) & 0x0F; unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1); unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1); enum { OP_COPY = 0, // Copy, used for things like to say it is <0,1,2,3> OP_VMRGHW, OP_VMRGLW, OP_VSPLTISW0, OP_VSPLTISW1, OP_VSPLTISW2, OP_VSPLTISW3, OP_VSLDOI4, OP_VSLDOI8, OP_VSLDOI12 }; if (OpNum == OP_COPY) { if (LHSID == (1*9+2)*9+3) return LHS; assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!"); return RHS; } SDValue OpLHS, OpRHS; OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl); OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl); int ShufIdxs[16]; switch (OpNum) { default: llvm_unreachable("Unknown i32 permute!"); case OP_VMRGHW: ShufIdxs[ 0] = 0; ShufIdxs[ 1] = 1; ShufIdxs[ 2] = 2; ShufIdxs[ 3] = 3; ShufIdxs[ 4] = 16; ShufIdxs[ 5] = 17; ShufIdxs[ 6] = 18; ShufIdxs[ 7] = 19; ShufIdxs[ 8] = 4; ShufIdxs[ 9] = 5; ShufIdxs[10] = 6; ShufIdxs[11] = 7; ShufIdxs[12] = 20; ShufIdxs[13] = 21; ShufIdxs[14] = 22; ShufIdxs[15] = 23; break; case OP_VMRGLW: ShufIdxs[ 0] = 8; ShufIdxs[ 1] = 9; ShufIdxs[ 2] = 10; ShufIdxs[ 3] = 11; ShufIdxs[ 4] = 24; ShufIdxs[ 5] = 25; ShufIdxs[ 6] = 26; ShufIdxs[ 7] = 27; ShufIdxs[ 8] = 12; ShufIdxs[ 9] = 13; ShufIdxs[10] = 14; ShufIdxs[11] = 15; ShufIdxs[12] = 28; ShufIdxs[13] = 29; ShufIdxs[14] = 30; ShufIdxs[15] = 31; break; case OP_VSPLTISW0: for (unsigned i = 0; i != 16; ++i) ShufIdxs[i] = (i&3)+0; break; case OP_VSPLTISW1: for (unsigned i = 0; i != 16; ++i) ShufIdxs[i] = (i&3)+4; break; case OP_VSPLTISW2: for (unsigned i = 0; i != 16; ++i) ShufIdxs[i] = (i&3)+8; break; case OP_VSPLTISW3: for (unsigned i = 0; i != 16; ++i) ShufIdxs[i] = (i&3)+12; break; case OP_VSLDOI4: return BuildVSLDOI(OpLHS, OpRHS, 4, OpLHS.getValueType(), DAG, dl); case OP_VSLDOI8: return BuildVSLDOI(OpLHS, OpRHS, 8, OpLHS.getValueType(), DAG, dl); case OP_VSLDOI12: return BuildVSLDOI(OpLHS, OpRHS, 12, OpLHS.getValueType(), DAG, dl); } EVT VT = OpLHS.getValueType(); OpLHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpLHS); OpRHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpRHS); SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, OpLHS, OpRHS, ShufIdxs); return DAG.getNode(ISD::BITCAST, dl, VT, T); } /// lowerToVINSERTB - Return the SDValue if this VECTOR_SHUFFLE can be handled /// by the VINSERTB instruction introduced in ISA 3.0, else just return default /// SDValue. SDValue PPCTargetLowering::lowerToVINSERTB(ShuffleVectorSDNode *N, SelectionDAG &DAG) const { const unsigned BytesInVector = 16; bool IsLE = Subtarget.isLittleEndian(); SDLoc dl(N); SDValue V1 = N->getOperand(0); SDValue V2 = N->getOperand(1); unsigned ShiftElts = 0, InsertAtByte = 0; bool Swap = false; // Shifts required to get the byte we want at element 7. unsigned LittleEndianShifts[] = {8, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9}; unsigned BigEndianShifts[] = {9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8}; ArrayRef Mask = N->getMask(); int OriginalOrder[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; // For each mask element, find out if we're just inserting something // from V2 into V1 or vice versa. // Possible permutations inserting an element from V2 into V1: // X, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 // 0, X, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 // ... // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, X // Inserting from V1 into V2 will be similar, except mask range will be // [16,31]. bool FoundCandidate = false; // If both vector operands for the shuffle are the same vector, the mask // will contain only elements from the first one and the second one will be // undef. unsigned VINSERTBSrcElem = IsLE ? 8 : 7; // Go through the mask of half-words to find an element that's being moved // from one vector to the other. for (unsigned i = 0; i < BytesInVector; ++i) { unsigned CurrentElement = Mask[i]; // If 2nd operand is undefined, we should only look for element 7 in the // Mask. if (V2.isUndef() && CurrentElement != VINSERTBSrcElem) continue; bool OtherElementsInOrder = true; // Examine the other elements in the Mask to see if they're in original // order. for (unsigned j = 0; j < BytesInVector; ++j) { if (j == i) continue; // If CurrentElement is from V1 [0,15], then we the rest of the Mask to be // from V2 [16,31] and vice versa. Unless the 2nd operand is undefined, // in which we always assume we're always picking from the 1st operand. int MaskOffset = (!V2.isUndef() && CurrentElement < BytesInVector) ? BytesInVector : 0; if (Mask[j] != OriginalOrder[j] + MaskOffset) { OtherElementsInOrder = false; break; } } // If other elements are in original order, we record the number of shifts // we need to get the element we want into element 7. Also record which byte // in the vector we should insert into. if (OtherElementsInOrder) { // If 2nd operand is undefined, we assume no shifts and no swapping. if (V2.isUndef()) { ShiftElts = 0; Swap = false; } else { // Only need the last 4-bits for shifts because operands will be swapped if CurrentElement is >= 2^4. ShiftElts = IsLE ? LittleEndianShifts[CurrentElement & 0xF] : BigEndianShifts[CurrentElement & 0xF]; Swap = CurrentElement < BytesInVector; } InsertAtByte = IsLE ? BytesInVector - (i + 1) : i; FoundCandidate = true; break; } } if (!FoundCandidate) return SDValue(); // Candidate found, construct the proper SDAG sequence with VINSERTB, // optionally with VECSHL if shift is required. if (Swap) std::swap(V1, V2); if (V2.isUndef()) V2 = V1; if (ShiftElts) { SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v16i8, V2, V2, DAG.getConstant(ShiftElts, dl, MVT::i32)); return DAG.getNode(PPCISD::VECINSERT, dl, MVT::v16i8, V1, Shl, DAG.getConstant(InsertAtByte, dl, MVT::i32)); } return DAG.getNode(PPCISD::VECINSERT, dl, MVT::v16i8, V1, V2, DAG.getConstant(InsertAtByte, dl, MVT::i32)); } /// lowerToVINSERTH - Return the SDValue if this VECTOR_SHUFFLE can be handled /// by the VINSERTH instruction introduced in ISA 3.0, else just return default /// SDValue. SDValue PPCTargetLowering::lowerToVINSERTH(ShuffleVectorSDNode *N, SelectionDAG &DAG) const { const unsigned NumHalfWords = 8; const unsigned BytesInVector = NumHalfWords * 2; // Check that the shuffle is on half-words. if (!isNByteElemShuffleMask(N, 2, 1)) return SDValue(); bool IsLE = Subtarget.isLittleEndian(); SDLoc dl(N); SDValue V1 = N->getOperand(0); SDValue V2 = N->getOperand(1); unsigned ShiftElts = 0, InsertAtByte = 0; bool Swap = false; // Shifts required to get the half-word we want at element 3. unsigned LittleEndianShifts[] = {4, 3, 2, 1, 0, 7, 6, 5}; unsigned BigEndianShifts[] = {5, 6, 7, 0, 1, 2, 3, 4}; uint32_t Mask = 0; uint32_t OriginalOrderLow = 0x1234567; uint32_t OriginalOrderHigh = 0x89ABCDEF; // Now we look at mask elements 0,2,4,6,8,10,12,14. Pack the mask into a // 32-bit space, only need 4-bit nibbles per element. for (unsigned i = 0; i < NumHalfWords; ++i) { unsigned MaskShift = (NumHalfWords - 1 - i) * 4; Mask |= ((uint32_t)(N->getMaskElt(i * 2) / 2) << MaskShift); } // For each mask element, find out if we're just inserting something // from V2 into V1 or vice versa. Possible permutations inserting an element // from V2 into V1: // X, 1, 2, 3, 4, 5, 6, 7 // 0, X, 2, 3, 4, 5, 6, 7 // 0, 1, X, 3, 4, 5, 6, 7 // 0, 1, 2, X, 4, 5, 6, 7 // 0, 1, 2, 3, X, 5, 6, 7 // 0, 1, 2, 3, 4, X, 6, 7 // 0, 1, 2, 3, 4, 5, X, 7 // 0, 1, 2, 3, 4, 5, 6, X // Inserting from V1 into V2 will be similar, except mask range will be [8,15]. bool FoundCandidate = false; // Go through the mask of half-words to find an element that's being moved // from one vector to the other. for (unsigned i = 0; i < NumHalfWords; ++i) { unsigned MaskShift = (NumHalfWords - 1 - i) * 4; uint32_t MaskOneElt = (Mask >> MaskShift) & 0xF; uint32_t MaskOtherElts = ~(0xF << MaskShift); uint32_t TargetOrder = 0x0; // If both vector operands for the shuffle are the same vector, the mask // will contain only elements from the first one and the second one will be // undef. if (V2.isUndef()) { ShiftElts = 0; unsigned VINSERTHSrcElem = IsLE ? 4 : 3; TargetOrder = OriginalOrderLow; Swap = false; // Skip if not the correct element or mask of other elements don't equal // to our expected order. if (MaskOneElt == VINSERTHSrcElem && (Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) { InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2; FoundCandidate = true; break; } } else { // If both operands are defined. // Target order is [8,15] if the current mask is between [0,7]. TargetOrder = (MaskOneElt < NumHalfWords) ? OriginalOrderHigh : OriginalOrderLow; // Skip if mask of other elements don't equal our expected order. if ((Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) { // We only need the last 3 bits for the number of shifts. ShiftElts = IsLE ? LittleEndianShifts[MaskOneElt & 0x7] : BigEndianShifts[MaskOneElt & 0x7]; InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2; Swap = MaskOneElt < NumHalfWords; FoundCandidate = true; break; } } } if (!FoundCandidate) return SDValue(); // Candidate found, construct the proper SDAG sequence with VINSERTH, // optionally with VECSHL if shift is required. if (Swap) std::swap(V1, V2); if (V2.isUndef()) V2 = V1; SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); if (ShiftElts) { // Double ShiftElts because we're left shifting on v16i8 type. SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v16i8, V2, V2, DAG.getConstant(2 * ShiftElts, dl, MVT::i32)); SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, Shl); SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v8i16, Conv1, Conv2, DAG.getConstant(InsertAtByte, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); } SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2); SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v8i16, Conv1, Conv2, DAG.getConstant(InsertAtByte, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); } /// LowerVECTOR_SHUFFLE - Return the code we lower for VECTOR_SHUFFLE. If this /// is a shuffle we can handle in a single instruction, return it. Otherwise, /// return the code it can be lowered into. Worst case, it can always be /// lowered into a vperm. SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); SDValue V1 = Op.getOperand(0); SDValue V2 = Op.getOperand(1); ShuffleVectorSDNode *SVOp = cast(Op); EVT VT = Op.getValueType(); bool isLittleEndian = Subtarget.isLittleEndian(); unsigned ShiftElts, InsertAtByte; bool Swap = false; if (Subtarget.hasP9Vector() && PPC::isXXINSERTWMask(SVOp, ShiftElts, InsertAtByte, Swap, isLittleEndian)) { if (Swap) std::swap(V1, V2); SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V2); if (ShiftElts) { SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v4i32, Conv2, Conv2, DAG.getConstant(ShiftElts, dl, MVT::i32)); SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v4i32, Conv1, Shl, DAG.getConstant(InsertAtByte, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); } SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v4i32, Conv1, Conv2, DAG.getConstant(InsertAtByte, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); } if (Subtarget.hasP9Altivec()) { SDValue NewISDNode; if ((NewISDNode = lowerToVINSERTH(SVOp, DAG))) return NewISDNode; if ((NewISDNode = lowerToVINSERTB(SVOp, DAG))) return NewISDNode; } if (Subtarget.hasVSX() && PPC::isXXSLDWIShuffleMask(SVOp, ShiftElts, Swap, isLittleEndian)) { if (Swap) std::swap(V1, V2); SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V2.isUndef() ? V1 : V2); SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v4i32, Conv1, Conv2, DAG.getConstant(ShiftElts, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Shl); } if (Subtarget.hasVSX() && PPC::isXXPERMDIShuffleMask(SVOp, ShiftElts, Swap, isLittleEndian)) { if (Swap) std::swap(V1, V2); SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1); SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V2.isUndef() ? V1 : V2); SDValue PermDI = DAG.getNode(PPCISD::XXPERMDI, dl, MVT::v2i64, Conv1, Conv2, DAG.getConstant(ShiftElts, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, PermDI); } if (Subtarget.hasP9Vector()) { if (PPC::isXXBRHShuffleMask(SVOp)) { SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); SDValue ReveHWord = DAG.getNode(PPCISD::XXREVERSE, dl, MVT::v8i16, Conv); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveHWord); } else if (PPC::isXXBRWShuffleMask(SVOp)) { SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); SDValue ReveWord = DAG.getNode(PPCISD::XXREVERSE, dl, MVT::v4i32, Conv); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveWord); } else if (PPC::isXXBRDShuffleMask(SVOp)) { SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1); SDValue ReveDWord = DAG.getNode(PPCISD::XXREVERSE, dl, MVT::v2i64, Conv); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveDWord); } else if (PPC::isXXBRQShuffleMask(SVOp)) { SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v1i128, V1); SDValue ReveQWord = DAG.getNode(PPCISD::XXREVERSE, dl, MVT::v1i128, Conv); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveQWord); } } if (Subtarget.hasVSX()) { if (V2.isUndef() && PPC::isSplatShuffleMask(SVOp, 4)) { int SplatIdx = PPC::getVSPLTImmediate(SVOp, 4, DAG); SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); SDValue Splat = DAG.getNode(PPCISD::XXSPLT, dl, MVT::v4i32, Conv, DAG.getConstant(SplatIdx, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Splat); } // Left shifts of 8 bytes are actually swaps. Convert accordingly. if (V2.isUndef() && PPC::isVSLDOIShuffleMask(SVOp, 1, DAG) == 8) { SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1); SDValue Swap = DAG.getNode(PPCISD::SWAP_NO_CHAIN, dl, MVT::v2f64, Conv); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Swap); } } if (Subtarget.hasQPX()) { if (VT.getVectorNumElements() != 4) return SDValue(); if (V2.isUndef()) V2 = V1; int AlignIdx = PPC::isQVALIGNIShuffleMask(SVOp); if (AlignIdx != -1) { return DAG.getNode(PPCISD::QVALIGNI, dl, VT, V1, V2, DAG.getConstant(AlignIdx, dl, MVT::i32)); } else if (SVOp->isSplat()) { int SplatIdx = SVOp->getSplatIndex(); if (SplatIdx >= 4) { std::swap(V1, V2); SplatIdx -= 4; } return DAG.getNode(PPCISD::QVESPLATI, dl, VT, V1, DAG.getConstant(SplatIdx, dl, MVT::i32)); } // Lower this into a qvgpci/qvfperm pair. // Compute the qvgpci literal unsigned idx = 0; for (unsigned i = 0; i < 4; ++i) { int m = SVOp->getMaskElt(i); unsigned mm = m >= 0 ? (unsigned) m : i; idx |= mm << (3-i)*3; } SDValue V3 = DAG.getNode(PPCISD::QVGPCI, dl, MVT::v4f64, DAG.getConstant(idx, dl, MVT::i32)); return DAG.getNode(PPCISD::QVFPERM, dl, VT, V1, V2, V3); } // Cases that are handled by instructions that take permute immediates // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be // selected by the instruction selector. if (V2.isUndef()) { if (PPC::isSplatShuffleMask(SVOp, 1) || PPC::isSplatShuffleMask(SVOp, 2) || PPC::isSplatShuffleMask(SVOp, 4) || PPC::isVPKUWUMShuffleMask(SVOp, 1, DAG) || PPC::isVPKUHUMShuffleMask(SVOp, 1, DAG) || PPC::isVSLDOIShuffleMask(SVOp, 1, DAG) != -1 || PPC::isVMRGLShuffleMask(SVOp, 1, 1, DAG) || PPC::isVMRGLShuffleMask(SVOp, 2, 1, DAG) || PPC::isVMRGLShuffleMask(SVOp, 4, 1, DAG) || PPC::isVMRGHShuffleMask(SVOp, 1, 1, DAG) || PPC::isVMRGHShuffleMask(SVOp, 2, 1, DAG) || PPC::isVMRGHShuffleMask(SVOp, 4, 1, DAG) || (Subtarget.hasP8Altivec() && ( PPC::isVPKUDUMShuffleMask(SVOp, 1, DAG) || PPC::isVMRGEOShuffleMask(SVOp, true, 1, DAG) || PPC::isVMRGEOShuffleMask(SVOp, false, 1, DAG)))) { return Op; } } // Altivec has a variety of "shuffle immediates" that take two vector inputs // and produce a fixed permutation. If any of these match, do not lower to // VPERM. unsigned int ShuffleKind = isLittleEndian ? 2 : 0; if (PPC::isVPKUWUMShuffleMask(SVOp, ShuffleKind, DAG) || PPC::isVPKUHUMShuffleMask(SVOp, ShuffleKind, DAG) || PPC::isVSLDOIShuffleMask(SVOp, ShuffleKind, DAG) != -1 || PPC::isVMRGLShuffleMask(SVOp, 1, ShuffleKind, DAG) || PPC::isVMRGLShuffleMask(SVOp, 2, ShuffleKind, DAG) || PPC::isVMRGLShuffleMask(SVOp, 4, ShuffleKind, DAG) || PPC::isVMRGHShuffleMask(SVOp, 1, ShuffleKind, DAG) || PPC::isVMRGHShuffleMask(SVOp, 2, ShuffleKind, DAG) || PPC::isVMRGHShuffleMask(SVOp, 4, ShuffleKind, DAG) || (Subtarget.hasP8Altivec() && ( PPC::isVPKUDUMShuffleMask(SVOp, ShuffleKind, DAG) || PPC::isVMRGEOShuffleMask(SVOp, true, ShuffleKind, DAG) || PPC::isVMRGEOShuffleMask(SVOp, false, ShuffleKind, DAG)))) return Op; // Check to see if this is a shuffle of 4-byte values. If so, we can use our // perfect shuffle table to emit an optimal matching sequence. ArrayRef PermMask = SVOp->getMask(); unsigned PFIndexes[4]; bool isFourElementShuffle = true; for (unsigned i = 0; i != 4 && isFourElementShuffle; ++i) { // Element number unsigned EltNo = 8; // Start out undef. for (unsigned j = 0; j != 4; ++j) { // Intra-element byte. if (PermMask[i*4+j] < 0) continue; // Undef, ignore it. unsigned ByteSource = PermMask[i*4+j]; if ((ByteSource & 3) != j) { isFourElementShuffle = false; break; } if (EltNo == 8) { EltNo = ByteSource/4; } else if (EltNo != ByteSource/4) { isFourElementShuffle = false; break; } } PFIndexes[i] = EltNo; } // If this shuffle can be expressed as a shuffle of 4-byte elements, use the // perfect shuffle vector to determine if it is cost effective to do this as // discrete instructions, or whether we should use a vperm. // For now, we skip this for little endian until such time as we have a // little-endian perfect shuffle table. if (isFourElementShuffle && !isLittleEndian) { // Compute the index in the perfect shuffle table. unsigned PFTableIndex = PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3]; unsigned PFEntry = PerfectShuffleTable[PFTableIndex]; unsigned Cost = (PFEntry >> 30); // Determining when to avoid vperm is tricky. Many things affect the cost // of vperm, particularly how many times the perm mask needs to be computed. // For example, if the perm mask can be hoisted out of a loop or is already // used (perhaps because there are multiple permutes with the same shuffle // mask?) the vperm has a cost of 1. OTOH, hoisting the permute mask out of // the loop requires an extra register. // // As a compromise, we only emit discrete instructions if the shuffle can be // generated in 3 or fewer operations. When we have loop information // available, if this block is within a loop, we should avoid using vperm // for 3-operation perms and use a constant pool load instead. if (Cost < 3) return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl); } // Lower this to a VPERM(V1, V2, V3) expression, where V3 is a constant // vector that will get spilled to the constant pool. if (V2.isUndef()) V2 = V1; // The SHUFFLE_VECTOR mask is almost exactly what we want for vperm, except // that it is in input element units, not in bytes. Convert now. // For little endian, the order of the input vectors is reversed, and // the permutation mask is complemented with respect to 31. This is // necessary to produce proper semantics with the big-endian-biased vperm // instruction. EVT EltVT = V1.getValueType().getVectorElementType(); unsigned BytesPerElement = EltVT.getSizeInBits()/8; SmallVector ResultMask; for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) { unsigned SrcElt = PermMask[i] < 0 ? 0 : PermMask[i]; for (unsigned j = 0; j != BytesPerElement; ++j) if (isLittleEndian) ResultMask.push_back(DAG.getConstant(31 - (SrcElt*BytesPerElement + j), dl, MVT::i32)); else ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement + j, dl, MVT::i32)); } SDValue VPermMask = DAG.getBuildVector(MVT::v16i8, dl, ResultMask); if (isLittleEndian) return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(), V2, V1, VPermMask); else return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(), V1, V2, VPermMask); } /// getVectorCompareInfo - Given an intrinsic, return false if it is not a /// vector comparison. If it is, return true and fill in Opc/isDot with /// information about the intrinsic. static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc, bool &isDot, const PPCSubtarget &Subtarget) { unsigned IntrinsicID = cast(Intrin.getOperand(0))->getZExtValue(); CompareOpc = -1; isDot = false; switch (IntrinsicID) { default: return false; // Comparison predicates. case Intrinsic::ppc_altivec_vcmpbfp_p: CompareOpc = 966; isDot = true; break; case Intrinsic::ppc_altivec_vcmpeqfp_p: CompareOpc = 198; isDot = true; break; case Intrinsic::ppc_altivec_vcmpequb_p: CompareOpc = 6; isDot = true; break; case Intrinsic::ppc_altivec_vcmpequh_p: CompareOpc = 70; isDot = true; break; case Intrinsic::ppc_altivec_vcmpequw_p: CompareOpc = 134; isDot = true; break; case Intrinsic::ppc_altivec_vcmpequd_p: if (Subtarget.hasP8Altivec()) { CompareOpc = 199; isDot = true; } else return false; break; case Intrinsic::ppc_altivec_vcmpneb_p: case Intrinsic::ppc_altivec_vcmpneh_p: case Intrinsic::ppc_altivec_vcmpnew_p: case Intrinsic::ppc_altivec_vcmpnezb_p: case Intrinsic::ppc_altivec_vcmpnezh_p: case Intrinsic::ppc_altivec_vcmpnezw_p: if (Subtarget.hasP9Altivec()) { switch (IntrinsicID) { default: llvm_unreachable("Unknown comparison intrinsic."); case Intrinsic::ppc_altivec_vcmpneb_p: CompareOpc = 7; break; case Intrinsic::ppc_altivec_vcmpneh_p: CompareOpc = 71; break; case Intrinsic::ppc_altivec_vcmpnew_p: CompareOpc = 135; break; case Intrinsic::ppc_altivec_vcmpnezb_p: CompareOpc = 263; break; case Intrinsic::ppc_altivec_vcmpnezh_p: CompareOpc = 327; break; case Intrinsic::ppc_altivec_vcmpnezw_p: CompareOpc = 391; break; } isDot = true; } else return false; break; case Intrinsic::ppc_altivec_vcmpgefp_p: CompareOpc = 454; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtfp_p: CompareOpc = 710; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtsb_p: CompareOpc = 774; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtsh_p: CompareOpc = 838; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtsw_p: CompareOpc = 902; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtsd_p: if (Subtarget.hasP8Altivec()) { CompareOpc = 967; isDot = true; } else return false; break; case Intrinsic::ppc_altivec_vcmpgtub_p: CompareOpc = 518; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtuh_p: CompareOpc = 582; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtuw_p: CompareOpc = 646; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtud_p: if (Subtarget.hasP8Altivec()) { CompareOpc = 711; isDot = true; } else return false; break; // VSX predicate comparisons use the same infrastructure case Intrinsic::ppc_vsx_xvcmpeqdp_p: case Intrinsic::ppc_vsx_xvcmpgedp_p: case Intrinsic::ppc_vsx_xvcmpgtdp_p: case Intrinsic::ppc_vsx_xvcmpeqsp_p: case Intrinsic::ppc_vsx_xvcmpgesp_p: case Intrinsic::ppc_vsx_xvcmpgtsp_p: if (Subtarget.hasVSX()) { switch (IntrinsicID) { case Intrinsic::ppc_vsx_xvcmpeqdp_p: CompareOpc = 99; break; case Intrinsic::ppc_vsx_xvcmpgedp_p: CompareOpc = 115; break; case Intrinsic::ppc_vsx_xvcmpgtdp_p: CompareOpc = 107; break; case Intrinsic::ppc_vsx_xvcmpeqsp_p: CompareOpc = 67; break; case Intrinsic::ppc_vsx_xvcmpgesp_p: CompareOpc = 83; break; case Intrinsic::ppc_vsx_xvcmpgtsp_p: CompareOpc = 75; break; } isDot = true; } else return false; break; // Normal Comparisons. case Intrinsic::ppc_altivec_vcmpbfp: CompareOpc = 966; break; case Intrinsic::ppc_altivec_vcmpeqfp: CompareOpc = 198; break; case Intrinsic::ppc_altivec_vcmpequb: CompareOpc = 6; break; case Intrinsic::ppc_altivec_vcmpequh: CompareOpc = 70; break; case Intrinsic::ppc_altivec_vcmpequw: CompareOpc = 134; break; case Intrinsic::ppc_altivec_vcmpequd: if (Subtarget.hasP8Altivec()) CompareOpc = 199; else return false; break; case Intrinsic::ppc_altivec_vcmpneb: case Intrinsic::ppc_altivec_vcmpneh: case Intrinsic::ppc_altivec_vcmpnew: case Intrinsic::ppc_altivec_vcmpnezb: case Intrinsic::ppc_altivec_vcmpnezh: case Intrinsic::ppc_altivec_vcmpnezw: if (Subtarget.hasP9Altivec()) switch (IntrinsicID) { default: llvm_unreachable("Unknown comparison intrinsic."); case Intrinsic::ppc_altivec_vcmpneb: CompareOpc = 7; break; case Intrinsic::ppc_altivec_vcmpneh: CompareOpc = 71; break; case Intrinsic::ppc_altivec_vcmpnew: CompareOpc = 135; break; case Intrinsic::ppc_altivec_vcmpnezb: CompareOpc = 263; break; case Intrinsic::ppc_altivec_vcmpnezh: CompareOpc = 327; break; case Intrinsic::ppc_altivec_vcmpnezw: CompareOpc = 391; break; } else return false; break; case Intrinsic::ppc_altivec_vcmpgefp: CompareOpc = 454; break; case Intrinsic::ppc_altivec_vcmpgtfp: CompareOpc = 710; break; case Intrinsic::ppc_altivec_vcmpgtsb: CompareOpc = 774; break; case Intrinsic::ppc_altivec_vcmpgtsh: CompareOpc = 838; break; case Intrinsic::ppc_altivec_vcmpgtsw: CompareOpc = 902; break; case Intrinsic::ppc_altivec_vcmpgtsd: if (Subtarget.hasP8Altivec()) CompareOpc = 967; else return false; break; case Intrinsic::ppc_altivec_vcmpgtub: CompareOpc = 518; break; case Intrinsic::ppc_altivec_vcmpgtuh: CompareOpc = 582; break; case Intrinsic::ppc_altivec_vcmpgtuw: CompareOpc = 646; break; case Intrinsic::ppc_altivec_vcmpgtud: if (Subtarget.hasP8Altivec()) CompareOpc = 711; else return false; break; } return true; } /// LowerINTRINSIC_WO_CHAIN - If this is an intrinsic that we want to custom /// lower, do it, otherwise return null. SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const { unsigned IntrinsicID = cast(Op.getOperand(0))->getZExtValue(); SDLoc dl(Op); if (IntrinsicID == Intrinsic::thread_pointer) { // Reads the thread pointer register, used for __builtin_thread_pointer. if (Subtarget.isPPC64()) return DAG.getRegister(PPC::X13, MVT::i64); return DAG.getRegister(PPC::R2, MVT::i32); } // If this is a lowered altivec predicate compare, CompareOpc is set to the // opcode number of the comparison. int CompareOpc; bool isDot; if (!getVectorCompareInfo(Op, CompareOpc, isDot, Subtarget)) return SDValue(); // Don't custom lower most intrinsics. // If this is a non-dot comparison, make the VCMP node and we are done. if (!isDot) { SDValue Tmp = DAG.getNode(PPCISD::VCMP, dl, Op.getOperand(2).getValueType(), Op.getOperand(1), Op.getOperand(2), DAG.getConstant(CompareOpc, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Tmp); } // Create the PPCISD altivec 'dot' comparison node. SDValue Ops[] = { Op.getOperand(2), // LHS Op.getOperand(3), // RHS DAG.getConstant(CompareOpc, dl, MVT::i32) }; EVT VTs[] = { Op.getOperand(2).getValueType(), MVT::Glue }; SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops); // Now that we have the comparison, emit a copy from the CR to a GPR. // This is flagged to the above dot comparison. SDValue Flags = DAG.getNode(PPCISD::MFOCRF, dl, MVT::i32, DAG.getRegister(PPC::CR6, MVT::i32), CompNode.getValue(1)); // Unpack the result based on how the target uses it. unsigned BitNo; // Bit # of CR6. bool InvertBit; // Invert result? switch (cast(Op.getOperand(1))->getZExtValue()) { default: // Can't happen, don't crash on invalid number though. case 0: // Return the value of the EQ bit of CR6. BitNo = 0; InvertBit = false; break; case 1: // Return the inverted value of the EQ bit of CR6. BitNo = 0; InvertBit = true; break; case 2: // Return the value of the LT bit of CR6. BitNo = 2; InvertBit = false; break; case 3: // Return the inverted value of the LT bit of CR6. BitNo = 2; InvertBit = true; break; } // Shift the bit into the low position. Flags = DAG.getNode(ISD::SRL, dl, MVT::i32, Flags, DAG.getConstant(8 - (3 - BitNo), dl, MVT::i32)); // Isolate the bit. Flags = DAG.getNode(ISD::AND, dl, MVT::i32, Flags, DAG.getConstant(1, dl, MVT::i32)); // If we are supposed to, toggle the bit. if (InvertBit) Flags = DAG.getNode(ISD::XOR, dl, MVT::i32, Flags, DAG.getConstant(1, dl, MVT::i32)); return Flags; } SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const { // SelectionDAGBuilder::visitTargetIntrinsic may insert one extra chain to // the beginning of the argument list. int ArgStart = isa(Op.getOperand(0)) ? 0 : 1; SDLoc DL(Op); switch (cast(Op.getOperand(ArgStart))->getZExtValue()) { case Intrinsic::ppc_cfence: { assert(ArgStart == 1 && "llvm.ppc.cfence must carry a chain argument."); assert(Subtarget.isPPC64() && "Only 64-bit is supported for now."); return SDValue(DAG.getMachineNode(PPC::CFENCE8, DL, MVT::Other, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(ArgStart + 1)), Op.getOperand(0)), 0); } default: break; } return SDValue(); } SDValue PPCTargetLowering::LowerREM(SDValue Op, SelectionDAG &DAG) const { // Check for a DIV with the same operands as this REM. for (auto UI : Op.getOperand(1)->uses()) { if ((Op.getOpcode() == ISD::SREM && UI->getOpcode() == ISD::SDIV) || (Op.getOpcode() == ISD::UREM && UI->getOpcode() == ISD::UDIV)) if (UI->getOperand(0) == Op.getOperand(0) && UI->getOperand(1) == Op.getOperand(1)) return SDValue(); } return Op; } // Lower scalar BSWAP64 to xxbrd. SDValue PPCTargetLowering::LowerBSWAP(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); // MTVSRDD Op = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i64, Op.getOperand(0), Op.getOperand(0)); // XXBRD Op = DAG.getNode(PPCISD::XXREVERSE, dl, MVT::v2i64, Op); // MFVSRD int VectorIndex = 0; if (Subtarget.isLittleEndian()) VectorIndex = 1; Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Op, DAG.getTargetConstant(VectorIndex, dl, MVT::i32)); return Op; } // ATOMIC_CMP_SWAP for i8/i16 needs to zero-extend its input since it will be // compared to a value that is atomically loaded (atomic loads zero-extend). SDValue PPCTargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const { assert(Op.getOpcode() == ISD::ATOMIC_CMP_SWAP && "Expecting an atomic compare-and-swap here."); SDLoc dl(Op); auto *AtomicNode = cast(Op.getNode()); EVT MemVT = AtomicNode->getMemoryVT(); if (MemVT.getSizeInBits() >= 32) return Op; SDValue CmpOp = Op.getOperand(2); // If this is already correctly zero-extended, leave it alone. auto HighBits = APInt::getHighBitsSet(32, 32 - MemVT.getSizeInBits()); if (DAG.MaskedValueIsZero(CmpOp, HighBits)) return Op; // Clear the high bits of the compare operand. unsigned MaskVal = (1 << MemVT.getSizeInBits()) - 1; SDValue NewCmpOp = DAG.getNode(ISD::AND, dl, MVT::i32, CmpOp, DAG.getConstant(MaskVal, dl, MVT::i32)); // Replace the existing compare operand with the properly zero-extended one. SmallVector Ops; for (int i = 0, e = AtomicNode->getNumOperands(); i < e; i++) Ops.push_back(AtomicNode->getOperand(i)); Ops[2] = NewCmpOp; MachineMemOperand *MMO = AtomicNode->getMemOperand(); SDVTList Tys = DAG.getVTList(MVT::i32, MVT::Other); auto NodeTy = (MemVT == MVT::i8) ? PPCISD::ATOMIC_CMP_SWAP_8 : PPCISD::ATOMIC_CMP_SWAP_16; return DAG.getMemIntrinsicNode(NodeTy, dl, Tys, Ops, MemVT, MMO); } SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); // Create a stack slot that is 16-byte aligned. MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); int FrameIdx = MFI.CreateStackObject(16, 16, false); EVT PtrVT = getPointerTy(DAG.getDataLayout()); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); // Store the input value into Value#0 of the stack slot. SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx, MachinePointerInfo()); // Load it out. return DAG.getLoad(Op.getValueType(), dl, Store, FIdx, MachinePointerInfo()); } SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const { assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT && "Should only be called for ISD::INSERT_VECTOR_ELT"); ConstantSDNode *C = dyn_cast(Op.getOperand(2)); // We have legal lowering for constant indices but not for variable ones. if (!C) return SDValue(); EVT VT = Op.getValueType(); SDLoc dl(Op); SDValue V1 = Op.getOperand(0); SDValue V2 = Op.getOperand(1); // We can use MTVSRZ + VECINSERT for v8i16 and v16i8 types. if (VT == MVT::v8i16 || VT == MVT::v16i8) { SDValue Mtvsrz = DAG.getNode(PPCISD::MTVSRZ, dl, VT, V2); unsigned BytesInEachElement = VT.getVectorElementType().getSizeInBits() / 8; unsigned InsertAtElement = C->getZExtValue(); unsigned InsertAtByte = InsertAtElement * BytesInEachElement; if (Subtarget.isLittleEndian()) { InsertAtByte = (16 - BytesInEachElement) - InsertAtByte; } return DAG.getNode(PPCISD::VECINSERT, dl, VT, V1, Mtvsrz, DAG.getConstant(InsertAtByte, dl, MVT::i32)); } return Op; } SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); SDNode *N = Op.getNode(); assert(N->getOperand(0).getValueType() == MVT::v4i1 && "Unknown extract_vector_elt type"); SDValue Value = N->getOperand(0); // The first part of this is like the store lowering except that we don't // need to track the chain. // The values are now known to be -1 (false) or 1 (true). To convert this // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to // understand how to form the extending load. SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64); Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); // Now convert to an integer and store. Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32), Value); MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); int FrameIdx = MFI.CreateStackObject(16, 16, false); MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); EVT PtrVT = getPointerTy(DAG.getDataLayout()); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); SDValue StoreChain = DAG.getEntryNode(); SDValue Ops[] = {StoreChain, DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32), Value, FIdx}; SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other); StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, dl, VTs, Ops, MVT::v4i32, PtrInfo); // Extract the value requested. unsigned Offset = 4*cast(N->getOperand(1))->getZExtValue(); SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); SDValue IntVal = DAG.getLoad(MVT::i32, dl, StoreChain, Idx, PtrInfo.getWithOffset(Offset)); if (!Subtarget.useCRBits()) return IntVal; return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, IntVal); } /// Lowering for QPX v4i1 loads SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); LoadSDNode *LN = cast(Op.getNode()); SDValue LoadChain = LN->getChain(); SDValue BasePtr = LN->getBasePtr(); if (Op.getValueType() == MVT::v4f64 || Op.getValueType() == MVT::v4f32) { EVT MemVT = LN->getMemoryVT(); unsigned Alignment = LN->getAlignment(); // If this load is properly aligned, then it is legal. if (Alignment >= MemVT.getStoreSize()) return Op; EVT ScalarVT = Op.getValueType().getScalarType(), ScalarMemVT = MemVT.getScalarType(); unsigned Stride = ScalarMemVT.getStoreSize(); SDValue Vals[4], LoadChains[4]; for (unsigned Idx = 0; Idx < 4; ++Idx) { SDValue Load; if (ScalarVT != ScalarMemVT) Load = DAG.getExtLoad(LN->getExtensionType(), dl, ScalarVT, LoadChain, BasePtr, LN->getPointerInfo().getWithOffset(Idx * Stride), ScalarMemVT, MinAlign(Alignment, Idx * Stride), LN->getMemOperand()->getFlags(), LN->getAAInfo()); else Load = DAG.getLoad(ScalarVT, dl, LoadChain, BasePtr, LN->getPointerInfo().getWithOffset(Idx * Stride), MinAlign(Alignment, Idx * Stride), LN->getMemOperand()->getFlags(), LN->getAAInfo()); if (Idx == 0 && LN->isIndexed()) { assert(LN->getAddressingMode() == ISD::PRE_INC && "Unknown addressing mode on vector load"); Load = DAG.getIndexedLoad(Load, dl, BasePtr, LN->getOffset(), LN->getAddressingMode()); } Vals[Idx] = Load; LoadChains[Idx] = Load.getValue(1); BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, DAG.getConstant(Stride, dl, BasePtr.getValueType())); } SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); SDValue Value = DAG.getBuildVector(Op.getValueType(), dl, Vals); if (LN->isIndexed()) { SDValue RetOps[] = { Value, Vals[0].getValue(1), TF }; return DAG.getMergeValues(RetOps, dl); } SDValue RetOps[] = { Value, TF }; return DAG.getMergeValues(RetOps, dl); } assert(Op.getValueType() == MVT::v4i1 && "Unknown load to lower"); assert(LN->isUnindexed() && "Indexed v4i1 loads are not supported"); // To lower v4i1 from a byte array, we load the byte elements of the // vector and then reuse the BUILD_VECTOR logic. SDValue VectElmts[4], VectElmtChains[4]; for (unsigned i = 0; i < 4; ++i) { SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType()); Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx); VectElmts[i] = DAG.getExtLoad( ISD::EXTLOAD, dl, MVT::i32, LoadChain, Idx, LN->getPointerInfo().getWithOffset(i), MVT::i8, /* Alignment = */ 1, LN->getMemOperand()->getFlags(), LN->getAAInfo()); VectElmtChains[i] = VectElmts[i].getValue(1); } LoadChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, VectElmtChains); SDValue Value = DAG.getBuildVector(MVT::v4i1, dl, VectElmts); SDValue RVals[] = { Value, LoadChain }; return DAG.getMergeValues(RVals, dl); } /// Lowering for QPX v4i1 stores SDValue PPCTargetLowering::LowerVectorStore(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); StoreSDNode *SN = cast(Op.getNode()); SDValue StoreChain = SN->getChain(); SDValue BasePtr = SN->getBasePtr(); SDValue Value = SN->getValue(); if (Value.getValueType() == MVT::v4f64 || Value.getValueType() == MVT::v4f32) { EVT MemVT = SN->getMemoryVT(); unsigned Alignment = SN->getAlignment(); // If this store is properly aligned, then it is legal. if (Alignment >= MemVT.getStoreSize()) return Op; EVT ScalarVT = Value.getValueType().getScalarType(), ScalarMemVT = MemVT.getScalarType(); unsigned Stride = ScalarMemVT.getStoreSize(); SDValue Stores[4]; for (unsigned Idx = 0; Idx < 4; ++Idx) { SDValue Ex = DAG.getNode( ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Value, DAG.getConstant(Idx, dl, getVectorIdxTy(DAG.getDataLayout()))); SDValue Store; if (ScalarVT != ScalarMemVT) Store = DAG.getTruncStore(StoreChain, dl, Ex, BasePtr, SN->getPointerInfo().getWithOffset(Idx * Stride), ScalarMemVT, MinAlign(Alignment, Idx * Stride), SN->getMemOperand()->getFlags(), SN->getAAInfo()); else Store = DAG.getStore(StoreChain, dl, Ex, BasePtr, SN->getPointerInfo().getWithOffset(Idx * Stride), MinAlign(Alignment, Idx * Stride), SN->getMemOperand()->getFlags(), SN->getAAInfo()); if (Idx == 0 && SN->isIndexed()) { assert(SN->getAddressingMode() == ISD::PRE_INC && "Unknown addressing mode on vector store"); Store = DAG.getIndexedStore(Store, dl, BasePtr, SN->getOffset(), SN->getAddressingMode()); } BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, DAG.getConstant(Stride, dl, BasePtr.getValueType())); Stores[Idx] = Store; } SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); if (SN->isIndexed()) { SDValue RetOps[] = { TF, Stores[0].getValue(1) }; return DAG.getMergeValues(RetOps, dl); } return TF; } assert(SN->isUnindexed() && "Indexed v4i1 stores are not supported"); assert(Value.getValueType() == MVT::v4i1 && "Unknown store to lower"); // The values are now known to be -1 (false) or 1 (true). To convert this // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to // understand how to form the extending load. SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64); Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); // Now convert to an integer and store. Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32), Value); MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); int FrameIdx = MFI.CreateStackObject(16, 16, false); MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); EVT PtrVT = getPointerTy(DAG.getDataLayout()); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); SDValue Ops[] = {StoreChain, DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32), Value, FIdx}; SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other); StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, dl, VTs, Ops, MVT::v4i32, PtrInfo); // Move data into the byte array. SDValue Loads[4], LoadChains[4]; for (unsigned i = 0; i < 4; ++i) { unsigned Offset = 4*i; SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); Loads[i] = DAG.getLoad(MVT::i32, dl, StoreChain, Idx, PtrInfo.getWithOffset(Offset)); LoadChains[i] = Loads[i].getValue(1); } StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); SDValue Stores[4]; for (unsigned i = 0; i < 4; ++i) { SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType()); Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx); Stores[i] = DAG.getTruncStore( StoreChain, dl, Loads[i], Idx, SN->getPointerInfo().getWithOffset(i), MVT::i8, /* Alignment = */ 1, SN->getMemOperand()->getFlags(), SN->getAAInfo()); } StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); return StoreChain; } SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); if (Op.getValueType() == MVT::v4i32) { SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); SDValue Zero = BuildSplatI( 0, 1, MVT::v4i32, DAG, dl); SDValue Neg16 = BuildSplatI(-16, 4, MVT::v4i32, DAG, dl);//+16 as shift amt. SDValue RHSSwap = // = vrlw RHS, 16 BuildIntrinsicOp(Intrinsic::ppc_altivec_vrlw, RHS, Neg16, DAG, dl); // Shrinkify inputs to v8i16. LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, LHS); RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHS); RHSSwap = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHSSwap); // Low parts multiplied together, generating 32-bit results (we ignore the // top parts). SDValue LoProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmulouh, LHS, RHS, DAG, dl, MVT::v4i32); SDValue HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmsumuhm, LHS, RHSSwap, Zero, DAG, dl, MVT::v4i32); // Shift the high parts up 16 bits. HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, HiProd, Neg16, DAG, dl); return DAG.getNode(ISD::ADD, dl, MVT::v4i32, LoProd, HiProd); } else if (Op.getValueType() == MVT::v8i16) { SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); SDValue Zero = BuildSplatI(0, 1, MVT::v8i16, DAG, dl); return BuildIntrinsicOp(Intrinsic::ppc_altivec_vmladduhm, LHS, RHS, Zero, DAG, dl); } else if (Op.getValueType() == MVT::v16i8) { SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); bool isLittleEndian = Subtarget.isLittleEndian(); // Multiply the even 8-bit parts, producing 16-bit sums. SDValue EvenParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuleub, LHS, RHS, DAG, dl, MVT::v8i16); EvenParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, EvenParts); // Multiply the odd 8-bit parts, producing 16-bit sums. SDValue OddParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuloub, LHS, RHS, DAG, dl, MVT::v8i16); OddParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OddParts); // Merge the results together. Because vmuleub and vmuloub are // instructions with a big-endian bias, we must reverse the // element numbering and reverse the meaning of "odd" and "even" // when generating little endian code. int Ops[16]; for (unsigned i = 0; i != 8; ++i) { if (isLittleEndian) { Ops[i*2 ] = 2*i; Ops[i*2+1] = 2*i+16; } else { Ops[i*2 ] = 2*i+1; Ops[i*2+1] = 2*i+1+16; } } if (isLittleEndian) return DAG.getVectorShuffle(MVT::v16i8, dl, OddParts, EvenParts, Ops); else return DAG.getVectorShuffle(MVT::v16i8, dl, EvenParts, OddParts, Ops); } else { llvm_unreachable("Unknown mul to lower!"); } } SDValue PPCTargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const { assert(Op.getOpcode() == ISD::ABS && "Should only be called for ISD::ABS"); EVT VT = Op.getValueType(); assert(VT.isVector() && "Only set vector abs as custom, scalar abs shouldn't reach here!"); assert((VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8) && "Unexpected vector element type!"); assert((VT != MVT::v2i64 || Subtarget.hasP8Altivec()) && "Current subtarget doesn't support smax v2i64!"); // For vector abs, it can be lowered to: // abs x // ==> // y = -x // smax(x, y) SDLoc dl(Op); SDValue X = Op.getOperand(0); SDValue Zero = DAG.getConstant(0, dl, VT); SDValue Y = DAG.getNode(ISD::SUB, dl, VT, Zero, X); // SMAX patch https://reviews.llvm.org/D47332 // hasn't landed yet, so use intrinsic first here. // TODO: Should use SMAX directly once SMAX patch landed Intrinsic::ID BifID = Intrinsic::ppc_altivec_vmaxsw; if (VT == MVT::v2i64) BifID = Intrinsic::ppc_altivec_vmaxsd; else if (VT == MVT::v8i16) BifID = Intrinsic::ppc_altivec_vmaxsh; else if (VT == MVT::v16i8) BifID = Intrinsic::ppc_altivec_vmaxsb; return BuildIntrinsicOp(BifID, X, Y, DAG, dl, VT); } // Custom lowering for fpext vf32 to v2f64 SDValue PPCTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const { assert(Op.getOpcode() == ISD::FP_EXTEND && "Should only be called for ISD::FP_EXTEND"); // We only want to custom lower an extend from v2f32 to v2f64. if (Op.getValueType() != MVT::v2f64 || Op.getOperand(0).getValueType() != MVT::v2f32) return SDValue(); SDLoc dl(Op); SDValue Op0 = Op.getOperand(0); switch (Op0.getOpcode()) { default: return SDValue(); case ISD::FADD: case ISD::FMUL: case ISD::FSUB: { SDValue NewLoad[2]; for (unsigned i = 0, ie = Op0.getNumOperands(); i != ie; ++i) { // Ensure both input are loads. SDValue LdOp = Op0.getOperand(i); if (LdOp.getOpcode() != ISD::LOAD) return SDValue(); // Generate new load node. LoadSDNode *LD = cast(LdOp); SDValue LoadOps[] = { LD->getChain(), LD->getBasePtr() }; NewLoad[i] = DAG.getMemIntrinsicNode(PPCISD::LD_VSX_LH, dl, DAG.getVTList(MVT::v4f32, MVT::Other), LoadOps, LD->getMemoryVT(), LD->getMemOperand()); } SDValue NewOp = DAG.getNode(Op0.getOpcode(), SDLoc(Op0), MVT::v4f32, NewLoad[0], NewLoad[1], Op0.getNode()->getFlags()); return DAG.getNode(PPCISD::FP_EXTEND_LH, dl, MVT::v2f64, NewOp); } case ISD::LOAD: { LoadSDNode *LD = cast(Op0); SDValue LoadOps[] = { LD->getChain(), LD->getBasePtr() }; SDValue NewLd = DAG.getMemIntrinsicNode(PPCISD::LD_VSX_LH, dl, DAG.getVTList(MVT::v4f32, MVT::Other), LoadOps, LD->getMemoryVT(), LD->getMemOperand()); return DAG.getNode(PPCISD::FP_EXTEND_LH, dl, MVT::v2f64, NewLd); } } llvm_unreachable("ERROR:Should return for all cases within swtich."); } /// LowerOperation - Provide custom lowering hooks for some operations. /// SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { switch (Op.getOpcode()) { default: llvm_unreachable("Wasn't expecting to be able to lower this!"); case ISD::ConstantPool: return LowerConstantPool(Op, DAG); case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); case ISD::JumpTable: return LowerJumpTable(Op, DAG); case ISD::SETCC: return LowerSETCC(Op, DAG); case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG); case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG); // Variable argument lowering. case ISD::VASTART: return LowerVASTART(Op, DAG); case ISD::VAARG: return LowerVAARG(Op, DAG); case ISD::VACOPY: return LowerVACOPY(Op, DAG); case ISD::STACKRESTORE: return LowerSTACKRESTORE(Op, DAG); case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); case ISD::GET_DYNAMIC_AREA_OFFSET: return LowerGET_DYNAMIC_AREA_OFFSET(Op, DAG); // Exception handling lowering. case ISD::EH_DWARF_CFA: return LowerEH_DWARF_CFA(Op, DAG); case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG); case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG); case ISD::LOAD: return LowerLOAD(Op, DAG); case ISD::STORE: return LowerSTORE(Op, DAG); case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG); case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); case ISD::FP_TO_UINT: case ISD::FP_TO_SINT: return LowerFP_TO_INT(Op, DAG, SDLoc(Op)); case ISD::UINT_TO_FP: case ISD::SINT_TO_FP: return LowerINT_TO_FP(Op, DAG); case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); // Lower 64-bit shifts. case ISD::SHL_PARTS: return LowerSHL_PARTS(Op, DAG); case ISD::SRL_PARTS: return LowerSRL_PARTS(Op, DAG); case ISD::SRA_PARTS: return LowerSRA_PARTS(Op, DAG); // Vector-related lowering. case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); case ISD::MUL: return LowerMUL(Op, DAG); case ISD::ABS: return LowerABS(Op, DAG); case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG); // For counter-based loop handling. case ISD::INTRINSIC_W_CHAIN: return SDValue(); case ISD::BITCAST: return LowerBITCAST(Op, DAG); // Frame & Return address. case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG); case ISD::SREM: case ISD::UREM: return LowerREM(Op, DAG); case ISD::BSWAP: return LowerBSWAP(Op, DAG); case ISD::ATOMIC_CMP_SWAP: return LowerATOMIC_CMP_SWAP(Op, DAG); } } void PPCTargetLowering::ReplaceNodeResults(SDNode *N, SmallVectorImpl&Results, SelectionDAG &DAG) const { SDLoc dl(N); switch (N->getOpcode()) { default: llvm_unreachable("Do not know how to custom type legalize this operation!"); case ISD::READCYCLECOUNTER: { SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); SDValue RTB = DAG.getNode(PPCISD::READ_TIME_BASE, dl, VTs, N->getOperand(0)); Results.push_back(RTB); Results.push_back(RTB.getValue(1)); Results.push_back(RTB.getValue(2)); break; } case ISD::INTRINSIC_W_CHAIN: { if (cast(N->getOperand(1))->getZExtValue() != Intrinsic::loop_decrement) break; assert(N->getValueType(0) == MVT::i1 && "Unexpected result type for CTR decrement intrinsic"); EVT SVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), N->getValueType(0)); SDVTList VTs = DAG.getVTList(SVT, MVT::Other); SDValue NewInt = DAG.getNode(N->getOpcode(), dl, VTs, N->getOperand(0), N->getOperand(1)); Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewInt)); Results.push_back(NewInt.getValue(1)); break; } case ISD::VAARG: { if (!Subtarget.isSVR4ABI() || Subtarget.isPPC64()) return; EVT VT = N->getValueType(0); if (VT == MVT::i64) { SDValue NewNode = LowerVAARG(SDValue(N, 1), DAG); Results.push_back(NewNode); Results.push_back(NewNode.getValue(1)); } return; } case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: // LowerFP_TO_INT() can only handle f32 and f64. if (N->getOperand(0).getValueType() == MVT::ppcf128) return; Results.push_back(LowerFP_TO_INT(SDValue(N, 0), DAG, dl)); return; case ISD::TRUNCATE: { EVT TrgVT = N->getValueType(0); if (TrgVT.isVector() && isOperationCustom(N->getOpcode(), TrgVT) && N->getOperand(0).getValueType().getSizeInBits() <= 128) Results.push_back(LowerTRUNCATEVector(SDValue(N, 0), DAG)); return; } case ISD::BITCAST: // Don't handle bitcast here. return; } } //===----------------------------------------------------------------------===// // Other Lowering Code //===----------------------------------------------------------------------===// static Instruction* callIntrinsic(IRBuilder<> &Builder, Intrinsic::ID Id) { Module *M = Builder.GetInsertBlock()->getParent()->getParent(); Function *Func = Intrinsic::getDeclaration(M, Id); return Builder.CreateCall(Func, {}); } // The mappings for emitLeading/TrailingFence is taken from // http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html Instruction *PPCTargetLowering::emitLeadingFence(IRBuilder<> &Builder, Instruction *Inst, AtomicOrdering Ord) const { if (Ord == AtomicOrdering::SequentiallyConsistent) return callIntrinsic(Builder, Intrinsic::ppc_sync); if (isReleaseOrStronger(Ord)) return callIntrinsic(Builder, Intrinsic::ppc_lwsync); return nullptr; } Instruction *PPCTargetLowering::emitTrailingFence(IRBuilder<> &Builder, Instruction *Inst, AtomicOrdering Ord) const { if (Inst->hasAtomicLoad() && isAcquireOrStronger(Ord)) { // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html and // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html // and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification. if (isa(Inst) && Subtarget.isPPC64()) return Builder.CreateCall( Intrinsic::getDeclaration( Builder.GetInsertBlock()->getParent()->getParent(), Intrinsic::ppc_cfence, {Inst->getType()}), {Inst}); // FIXME: Can use isync for rmw operation. return callIntrinsic(Builder, Intrinsic::ppc_lwsync); } return nullptr; } MachineBasicBlock * PPCTargetLowering::EmitAtomicBinary(MachineInstr &MI, MachineBasicBlock *BB, unsigned AtomicSize, unsigned BinOpcode, unsigned CmpOpcode, unsigned CmpPred) const { // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. const TargetInstrInfo *TII = Subtarget.getInstrInfo(); auto LoadMnemonic = PPC::LDARX; auto StoreMnemonic = PPC::STDCX; switch (AtomicSize) { default: llvm_unreachable("Unexpected size of atomic entity"); case 1: LoadMnemonic = PPC::LBARX; StoreMnemonic = PPC::STBCX; assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4"); break; case 2: LoadMnemonic = PPC::LHARX; StoreMnemonic = PPC::STHCX; assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4"); break; case 4: LoadMnemonic = PPC::LWARX; StoreMnemonic = PPC::STWCX; break; case 8: LoadMnemonic = PPC::LDARX; StoreMnemonic = PPC::STDCX; break; } const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction *F = BB->getParent(); MachineFunction::iterator It = ++BB->getIterator(); Register dest = MI.getOperand(0).getReg(); Register ptrA = MI.getOperand(1).getReg(); Register ptrB = MI.getOperand(2).getReg(); Register incr = MI.getOperand(3).getReg(); DebugLoc dl = MI.getDebugLoc(); MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *loop2MBB = CmpOpcode ? F->CreateMachineBasicBlock(LLVM_BB) : nullptr; MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); F->insert(It, loopMBB); if (CmpOpcode) F->insert(It, loop2MBB); F->insert(It, exitMBB); exitMBB->splice(exitMBB->begin(), BB, std::next(MachineBasicBlock::iterator(MI)), BB->end()); exitMBB->transferSuccessorsAndUpdatePHIs(BB); MachineRegisterInfo &RegInfo = F->getRegInfo(); Register TmpReg = (!BinOpcode) ? incr : RegInfo.createVirtualRegister( AtomicSize == 8 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass); // thisMBB: // ... // fallthrough --> loopMBB BB->addSuccessor(loopMBB); // loopMBB: // l[wd]arx dest, ptr // add r0, dest, incr // st[wd]cx. r0, ptr // bne- loopMBB // fallthrough --> exitMBB // For max/min... // loopMBB: // l[wd]arx dest, ptr // cmpl?[wd] incr, dest // bgt exitMBB // loop2MBB: // st[wd]cx. dest, ptr // bne- loopMBB // fallthrough --> exitMBB BB = loopMBB; BuildMI(BB, dl, TII->get(LoadMnemonic), dest) .addReg(ptrA).addReg(ptrB); if (BinOpcode) BuildMI(BB, dl, TII->get(BinOpcode), TmpReg).addReg(incr).addReg(dest); if (CmpOpcode) { // Signed comparisons of byte or halfword values must be sign-extended. if (CmpOpcode == PPC::CMPW && AtomicSize < 4) { unsigned ExtReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass); BuildMI(BB, dl, TII->get(AtomicSize == 1 ? PPC::EXTSB : PPC::EXTSH), ExtReg).addReg(dest); BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0) .addReg(incr).addReg(ExtReg); } else BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0) .addReg(incr).addReg(dest); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(CmpPred).addReg(PPC::CR0).addMBB(exitMBB); BB->addSuccessor(loop2MBB); BB->addSuccessor(exitMBB); BB = loop2MBB; } BuildMI(BB, dl, TII->get(StoreMnemonic)) .addReg(TmpReg).addReg(ptrA).addReg(ptrB); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loopMBB); BB->addSuccessor(loopMBB); BB->addSuccessor(exitMBB); // exitMBB: // ... BB = exitMBB; return BB; } MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary( MachineInstr &MI, MachineBasicBlock *BB, bool is8bit, // operation unsigned BinOpcode, unsigned CmpOpcode, unsigned CmpPred) const { // If we support part-word atomic mnemonics, just use them if (Subtarget.hasPartwordAtomics()) return EmitAtomicBinary(MI, BB, is8bit ? 1 : 2, BinOpcode, CmpOpcode, CmpPred); // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. const TargetInstrInfo *TII = Subtarget.getInstrInfo(); // In 64 bit mode we have to use 64 bits for addresses, even though the // lwarx/stwcx are 32 bits. With the 32-bit atomics we can use address // registers without caring whether they're 32 or 64, but here we're // doing actual arithmetic on the addresses. bool is64bit = Subtarget.isPPC64(); bool isLittleEndian = Subtarget.isLittleEndian(); unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO; const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction *F = BB->getParent(); MachineFunction::iterator It = ++BB->getIterator(); unsigned dest = MI.getOperand(0).getReg(); unsigned ptrA = MI.getOperand(1).getReg(); unsigned ptrB = MI.getOperand(2).getReg(); unsigned incr = MI.getOperand(3).getReg(); DebugLoc dl = MI.getDebugLoc(); MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *loop2MBB = CmpOpcode ? F->CreateMachineBasicBlock(LLVM_BB) : nullptr; MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); F->insert(It, loopMBB); if (CmpOpcode) F->insert(It, loop2MBB); F->insert(It, exitMBB); exitMBB->splice(exitMBB->begin(), BB, std::next(MachineBasicBlock::iterator(MI)), BB->end()); exitMBB->transferSuccessorsAndUpdatePHIs(BB); MachineRegisterInfo &RegInfo = F->getRegInfo(); const TargetRegisterClass *RC = is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; const TargetRegisterClass *GPRC = &PPC::GPRCRegClass; Register PtrReg = RegInfo.createVirtualRegister(RC); Register Shift1Reg = RegInfo.createVirtualRegister(GPRC); Register ShiftReg = isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(GPRC); Register Incr2Reg = RegInfo.createVirtualRegister(GPRC); Register MaskReg = RegInfo.createVirtualRegister(GPRC); Register Mask2Reg = RegInfo.createVirtualRegister(GPRC); Register Mask3Reg = RegInfo.createVirtualRegister(GPRC); Register Tmp2Reg = RegInfo.createVirtualRegister(GPRC); Register Tmp3Reg = RegInfo.createVirtualRegister(GPRC); Register Tmp4Reg = RegInfo.createVirtualRegister(GPRC); Register TmpDestReg = RegInfo.createVirtualRegister(GPRC); Register Ptr1Reg; Register TmpReg = (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(GPRC); // thisMBB: // ... // fallthrough --> loopMBB BB->addSuccessor(loopMBB); // The 4-byte load must be aligned, while a char or short may be // anywhere in the word. Hence all this nasty bookkeeping code. // add ptr1, ptrA, ptrB [copy if ptrA==0] // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27] // xori shift, shift1, 24 [16] // rlwinm ptr, ptr1, 0, 0, 29 // slw incr2, incr, shift // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535] // slw mask, mask2, shift // loopMBB: // lwarx tmpDest, ptr // add tmp, tmpDest, incr2 // andc tmp2, tmpDest, mask // and tmp3, tmp, mask // or tmp4, tmp3, tmp2 // stwcx. tmp4, ptr // bne- loopMBB // fallthrough --> exitMBB // srw dest, tmpDest, shift if (ptrA != ZeroReg) { Ptr1Reg = RegInfo.createVirtualRegister(RC); BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg) .addReg(ptrA) .addReg(ptrB); } else { Ptr1Reg = ptrB; } // We need use 32-bit subregister to avoid mismatch register class in 64-bit // mode. BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg) .addReg(Ptr1Reg, 0, is64bit ? PPC::sub_32 : 0) .addImm(3) .addImm(27) .addImm(is8bit ? 28 : 27); if (!isLittleEndian) BuildMI(BB, dl, TII->get(PPC::XORI), ShiftReg) .addReg(Shift1Reg) .addImm(is8bit ? 24 : 16); if (is64bit) BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg) .addReg(Ptr1Reg) .addImm(0) .addImm(61); else BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg) .addReg(Ptr1Reg) .addImm(0) .addImm(0) .addImm(29); BuildMI(BB, dl, TII->get(PPC::SLW), Incr2Reg).addReg(incr).addReg(ShiftReg); if (is8bit) BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255); else { BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0); BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg) .addReg(Mask3Reg) .addImm(65535); } BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg) .addReg(Mask2Reg) .addReg(ShiftReg); BB = loopMBB; BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg) .addReg(ZeroReg) .addReg(PtrReg); if (BinOpcode) BuildMI(BB, dl, TII->get(BinOpcode), TmpReg) .addReg(Incr2Reg) .addReg(TmpDestReg); BuildMI(BB, dl, TII->get(PPC::ANDC), Tmp2Reg) .addReg(TmpDestReg) .addReg(MaskReg); BuildMI(BB, dl, TII->get(PPC::AND), Tmp3Reg).addReg(TmpReg).addReg(MaskReg); if (CmpOpcode) { // For unsigned comparisons, we can directly compare the shifted values. // For signed comparisons we shift and sign extend. unsigned SReg = RegInfo.createVirtualRegister(GPRC); BuildMI(BB, dl, TII->get(PPC::AND), SReg) .addReg(TmpDestReg) .addReg(MaskReg); unsigned ValueReg = SReg; unsigned CmpReg = Incr2Reg; if (CmpOpcode == PPC::CMPW) { ValueReg = RegInfo.createVirtualRegister(GPRC); BuildMI(BB, dl, TII->get(PPC::SRW), ValueReg) .addReg(SReg) .addReg(ShiftReg); unsigned ValueSReg = RegInfo.createVirtualRegister(GPRC); BuildMI(BB, dl, TII->get(is8bit ? PPC::EXTSB : PPC::EXTSH), ValueSReg) .addReg(ValueReg); ValueReg = ValueSReg; CmpReg = incr; } BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0) .addReg(CmpReg) .addReg(ValueReg); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(CmpPred) .addReg(PPC::CR0) .addMBB(exitMBB); BB->addSuccessor(loop2MBB); BB->addSuccessor(exitMBB); BB = loop2MBB; } BuildMI(BB, dl, TII->get(PPC::OR), Tmp4Reg).addReg(Tmp3Reg).addReg(Tmp2Reg); BuildMI(BB, dl, TII->get(PPC::STWCX)) .addReg(Tmp4Reg) .addReg(ZeroReg) .addReg(PtrReg); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(PPC::PRED_NE) .addReg(PPC::CR0) .addMBB(loopMBB); BB->addSuccessor(loopMBB); BB->addSuccessor(exitMBB); // exitMBB: // ... BB = exitMBB; BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest) .addReg(TmpDestReg) .addReg(ShiftReg); return BB; } llvm::MachineBasicBlock * PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI, MachineBasicBlock *MBB) const { DebugLoc DL = MI.getDebugLoc(); const TargetInstrInfo *TII = Subtarget.getInstrInfo(); const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo(); MachineFunction *MF = MBB->getParent(); MachineRegisterInfo &MRI = MF->getRegInfo(); const BasicBlock *BB = MBB->getBasicBlock(); MachineFunction::iterator I = ++MBB->getIterator(); unsigned DstReg = MI.getOperand(0).getReg(); const TargetRegisterClass *RC = MRI.getRegClass(DstReg); assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!"); unsigned mainDstReg = MRI.createVirtualRegister(RC); unsigned restoreDstReg = MRI.createVirtualRegister(RC); MVT PVT = getPointerTy(MF->getDataLayout()); assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!"); // For v = setjmp(buf), we generate // // thisMBB: // SjLjSetup mainMBB // bl mainMBB // v_restore = 1 // b sinkMBB // // mainMBB: // buf[LabelOffset] = LR // v_main = 0 // // sinkMBB: // v = phi(main, restore) // MachineBasicBlock *thisMBB = MBB; MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB); MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB); MF->insert(I, mainMBB); MF->insert(I, sinkMBB); MachineInstrBuilder MIB; // Transfer the remainder of BB and its successor edges to sinkMBB. sinkMBB->splice(sinkMBB->begin(), MBB, std::next(MachineBasicBlock::iterator(MI)), MBB->end()); sinkMBB->transferSuccessorsAndUpdatePHIs(MBB); // Note that the structure of the jmp_buf used here is not compatible // with that used by libc, and is not designed to be. Specifically, it // stores only those 'reserved' registers that LLVM does not otherwise // understand how to spill. Also, by convention, by the time this // intrinsic is called, Clang has already stored the frame address in the // first slot of the buffer and stack address in the third. Following the // X86 target code, we'll store the jump address in the second slot. We also // need to save the TOC pointer (R2) to handle jumps between shared // libraries, and that will be stored in the fourth slot. The thread // identifier (R13) is not affected. // thisMBB: const int64_t LabelOffset = 1 * PVT.getStoreSize(); const int64_t TOCOffset = 3 * PVT.getStoreSize(); const int64_t BPOffset = 4 * PVT.getStoreSize(); // Prepare IP either in reg. const TargetRegisterClass *PtrRC = getRegClassFor(PVT); unsigned LabelReg = MRI.createVirtualRegister(PtrRC); unsigned BufReg = MI.getOperand(1).getReg(); if (Subtarget.isPPC64() && Subtarget.isSVR4ABI()) { setUsesTOCBasePtr(*MBB->getParent()); MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::STD)) .addReg(PPC::X2) .addImm(TOCOffset) .addReg(BufReg) .cloneMemRefs(MI); } // Naked functions never have a base pointer, and so we use r1. For all // other functions, this decision must be delayed until during PEI. unsigned BaseReg; if (MF->getFunction().hasFnAttribute(Attribute::Naked)) BaseReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1; else BaseReg = Subtarget.isPPC64() ? PPC::BP8 : PPC::BP; MIB = BuildMI(*thisMBB, MI, DL, TII->get(Subtarget.isPPC64() ? PPC::STD : PPC::STW)) .addReg(BaseReg) .addImm(BPOffset) .addReg(BufReg) .cloneMemRefs(MI); // Setup MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::BCLalways)).addMBB(mainMBB); MIB.addRegMask(TRI->getNoPreservedMask()); BuildMI(*thisMBB, MI, DL, TII->get(PPC::LI), restoreDstReg).addImm(1); MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::EH_SjLj_Setup)) .addMBB(mainMBB); MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::B)).addMBB(sinkMBB); thisMBB->addSuccessor(mainMBB, BranchProbability::getZero()); thisMBB->addSuccessor(sinkMBB, BranchProbability::getOne()); // mainMBB: // mainDstReg = 0 MIB = BuildMI(mainMBB, DL, TII->get(Subtarget.isPPC64() ? PPC::MFLR8 : PPC::MFLR), LabelReg); // Store IP if (Subtarget.isPPC64()) { MIB = BuildMI(mainMBB, DL, TII->get(PPC::STD)) .addReg(LabelReg) .addImm(LabelOffset) .addReg(BufReg); } else { MIB = BuildMI(mainMBB, DL, TII->get(PPC::STW)) .addReg(LabelReg) .addImm(LabelOffset) .addReg(BufReg); } MIB.cloneMemRefs(MI); BuildMI(mainMBB, DL, TII->get(PPC::LI), mainDstReg).addImm(0); mainMBB->addSuccessor(sinkMBB); // sinkMBB: BuildMI(*sinkMBB, sinkMBB->begin(), DL, TII->get(PPC::PHI), DstReg) .addReg(mainDstReg).addMBB(mainMBB) .addReg(restoreDstReg).addMBB(thisMBB); MI.eraseFromParent(); return sinkMBB; } MachineBasicBlock * PPCTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI, MachineBasicBlock *MBB) const { DebugLoc DL = MI.getDebugLoc(); const TargetInstrInfo *TII = Subtarget.getInstrInfo(); MachineFunction *MF = MBB->getParent(); MachineRegisterInfo &MRI = MF->getRegInfo(); MVT PVT = getPointerTy(MF->getDataLayout()); assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!"); const TargetRegisterClass *RC = (PVT == MVT::i64) ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; unsigned Tmp = MRI.createVirtualRegister(RC); // Since FP is only updated here but NOT referenced, it's treated as GPR. unsigned FP = (PVT == MVT::i64) ? PPC::X31 : PPC::R31; unsigned SP = (PVT == MVT::i64) ? PPC::X1 : PPC::R1; unsigned BP = (PVT == MVT::i64) ? PPC::X30 : (Subtarget.isSVR4ABI() && isPositionIndependent() ? PPC::R29 : PPC::R30); MachineInstrBuilder MIB; const int64_t LabelOffset = 1 * PVT.getStoreSize(); const int64_t SPOffset = 2 * PVT.getStoreSize(); const int64_t TOCOffset = 3 * PVT.getStoreSize(); const int64_t BPOffset = 4 * PVT.getStoreSize(); unsigned BufReg = MI.getOperand(0).getReg(); // Reload FP (the jumped-to function may not have had a // frame pointer, and if so, then its r31 will be restored // as necessary). if (PVT == MVT::i64) { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), FP) .addImm(0) .addReg(BufReg); } else { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), FP) .addImm(0) .addReg(BufReg); } MIB.cloneMemRefs(MI); // Reload IP if (PVT == MVT::i64) { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), Tmp) .addImm(LabelOffset) .addReg(BufReg); } else { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), Tmp) .addImm(LabelOffset) .addReg(BufReg); } MIB.cloneMemRefs(MI); // Reload SP if (PVT == MVT::i64) { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), SP) .addImm(SPOffset) .addReg(BufReg); } else { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), SP) .addImm(SPOffset) .addReg(BufReg); } MIB.cloneMemRefs(MI); // Reload BP if (PVT == MVT::i64) { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), BP) .addImm(BPOffset) .addReg(BufReg); } else { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), BP) .addImm(BPOffset) .addReg(BufReg); } MIB.cloneMemRefs(MI); // Reload TOC if (PVT == MVT::i64 && Subtarget.isSVR4ABI()) { setUsesTOCBasePtr(*MBB->getParent()); MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), PPC::X2) .addImm(TOCOffset) .addReg(BufReg) .cloneMemRefs(MI); } // Jump BuildMI(*MBB, MI, DL, TII->get(PVT == MVT::i64 ? PPC::MTCTR8 : PPC::MTCTR)).addReg(Tmp); BuildMI(*MBB, MI, DL, TII->get(PVT == MVT::i64 ? PPC::BCTR8 : PPC::BCTR)); MI.eraseFromParent(); return MBB; } MachineBasicBlock * PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const { if (MI.getOpcode() == TargetOpcode::STACKMAP || MI.getOpcode() == TargetOpcode::PATCHPOINT) { if (Subtarget.isPPC64() && Subtarget.isSVR4ABI() && MI.getOpcode() == TargetOpcode::PATCHPOINT) { // Call lowering should have added an r2 operand to indicate a dependence // on the TOC base pointer value. It can't however, because there is no // way to mark the dependence as implicit there, and so the stackmap code // will confuse it with a regular operand. Instead, add the dependence // here. MI.addOperand(MachineOperand::CreateReg(PPC::X2, false, true)); } return emitPatchPoint(MI, BB); } if (MI.getOpcode() == PPC::EH_SjLj_SetJmp32 || MI.getOpcode() == PPC::EH_SjLj_SetJmp64) { return emitEHSjLjSetJmp(MI, BB); } else if (MI.getOpcode() == PPC::EH_SjLj_LongJmp32 || MI.getOpcode() == PPC::EH_SjLj_LongJmp64) { return emitEHSjLjLongJmp(MI, BB); } const TargetInstrInfo *TII = Subtarget.getInstrInfo(); // To "insert" these instructions we actually have to insert their // control-flow patterns. const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction::iterator It = ++BB->getIterator(); MachineFunction *F = BB->getParent(); if (MI.getOpcode() == PPC::SELECT_CC_I4 || MI.getOpcode() == PPC::SELECT_CC_I8 || MI.getOpcode() == PPC::SELECT_I4 || MI.getOpcode() == PPC::SELECT_I8) { SmallVector Cond; if (MI.getOpcode() == PPC::SELECT_CC_I4 || MI.getOpcode() == PPC::SELECT_CC_I8) Cond.push_back(MI.getOperand(4)); else Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET)); Cond.push_back(MI.getOperand(1)); DebugLoc dl = MI.getDebugLoc(); TII->insertSelect(*BB, MI, dl, MI.getOperand(0).getReg(), Cond, MI.getOperand(2).getReg(), MI.getOperand(3).getReg()); } else if (MI.getOpcode() == PPC::SELECT_CC_I4 || MI.getOpcode() == PPC::SELECT_CC_I8 || MI.getOpcode() == PPC::SELECT_CC_F4 || MI.getOpcode() == PPC::SELECT_CC_F8 || MI.getOpcode() == PPC::SELECT_CC_F16 || MI.getOpcode() == PPC::SELECT_CC_QFRC || MI.getOpcode() == PPC::SELECT_CC_QSRC || MI.getOpcode() == PPC::SELECT_CC_QBRC || MI.getOpcode() == PPC::SELECT_CC_VRRC || MI.getOpcode() == PPC::SELECT_CC_VSFRC || MI.getOpcode() == PPC::SELECT_CC_VSSRC || MI.getOpcode() == PPC::SELECT_CC_VSRC || MI.getOpcode() == PPC::SELECT_CC_SPE4 || MI.getOpcode() == PPC::SELECT_CC_SPE || MI.getOpcode() == PPC::SELECT_I4 || MI.getOpcode() == PPC::SELECT_I8 || MI.getOpcode() == PPC::SELECT_F4 || MI.getOpcode() == PPC::SELECT_F8 || MI.getOpcode() == PPC::SELECT_F16 || MI.getOpcode() == PPC::SELECT_QFRC || MI.getOpcode() == PPC::SELECT_QSRC || MI.getOpcode() == PPC::SELECT_QBRC || MI.getOpcode() == PPC::SELECT_SPE || MI.getOpcode() == PPC::SELECT_SPE4 || MI.getOpcode() == PPC::SELECT_VRRC || MI.getOpcode() == PPC::SELECT_VSFRC || MI.getOpcode() == PPC::SELECT_VSSRC || MI.getOpcode() == PPC::SELECT_VSRC) { // The incoming instruction knows the destination vreg to set, the // condition code register to branch on, the true/false values to // select between, and a branch opcode to use. // thisMBB: // ... // TrueVal = ... // cmpTY ccX, r1, r2 // bCC copy1MBB // fallthrough --> copy0MBB MachineBasicBlock *thisMBB = BB; MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); DebugLoc dl = MI.getDebugLoc(); F->insert(It, copy0MBB); F->insert(It, sinkMBB); // Transfer the remainder of BB and its successor edges to sinkMBB. sinkMBB->splice(sinkMBB->begin(), BB, std::next(MachineBasicBlock::iterator(MI)), BB->end()); sinkMBB->transferSuccessorsAndUpdatePHIs(BB); // Next, add the true and fallthrough blocks as its successors. BB->addSuccessor(copy0MBB); BB->addSuccessor(sinkMBB); if (MI.getOpcode() == PPC::SELECT_I4 || MI.getOpcode() == PPC::SELECT_I8 || MI.getOpcode() == PPC::SELECT_F4 || MI.getOpcode() == PPC::SELECT_F8 || MI.getOpcode() == PPC::SELECT_F16 || MI.getOpcode() == PPC::SELECT_SPE4 || MI.getOpcode() == PPC::SELECT_SPE || MI.getOpcode() == PPC::SELECT_QFRC || MI.getOpcode() == PPC::SELECT_QSRC || MI.getOpcode() == PPC::SELECT_QBRC || MI.getOpcode() == PPC::SELECT_VRRC || MI.getOpcode() == PPC::SELECT_VSFRC || MI.getOpcode() == PPC::SELECT_VSSRC || MI.getOpcode() == PPC::SELECT_VSRC) { BuildMI(BB, dl, TII->get(PPC::BC)) .addReg(MI.getOperand(1).getReg()) .addMBB(sinkMBB); } else { unsigned SelectPred = MI.getOperand(4).getImm(); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(SelectPred) .addReg(MI.getOperand(1).getReg()) .addMBB(sinkMBB); } // copy0MBB: // %FalseValue = ... // # fallthrough to sinkMBB BB = copy0MBB; // Update machine-CFG edges BB->addSuccessor(sinkMBB); // sinkMBB: // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] // ... BB = sinkMBB; BuildMI(*BB, BB->begin(), dl, TII->get(PPC::PHI), MI.getOperand(0).getReg()) .addReg(MI.getOperand(3).getReg()) .addMBB(copy0MBB) .addReg(MI.getOperand(2).getReg()) .addMBB(thisMBB); } else if (MI.getOpcode() == PPC::ReadTB) { // To read the 64-bit time-base register on a 32-bit target, we read the // two halves. Should the counter have wrapped while it was being read, we // need to try again. // ... // readLoop: // mfspr Rx,TBU # load from TBU // mfspr Ry,TB # load from TB // mfspr Rz,TBU # load from TBU // cmpw crX,Rx,Rz # check if 'old'='new' // bne readLoop # branch if they're not equal // ... MachineBasicBlock *readMBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); DebugLoc dl = MI.getDebugLoc(); F->insert(It, readMBB); F->insert(It, sinkMBB); // Transfer the remainder of BB and its successor edges to sinkMBB. sinkMBB->splice(sinkMBB->begin(), BB, std::next(MachineBasicBlock::iterator(MI)), BB->end()); sinkMBB->transferSuccessorsAndUpdatePHIs(BB); BB->addSuccessor(readMBB); BB = readMBB; MachineRegisterInfo &RegInfo = F->getRegInfo(); unsigned ReadAgainReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass); unsigned LoReg = MI.getOperand(0).getReg(); unsigned HiReg = MI.getOperand(1).getReg(); BuildMI(BB, dl, TII->get(PPC::MFSPR), HiReg).addImm(269); BuildMI(BB, dl, TII->get(PPC::MFSPR), LoReg).addImm(268); BuildMI(BB, dl, TII->get(PPC::MFSPR), ReadAgainReg).addImm(269); unsigned CmpReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass); BuildMI(BB, dl, TII->get(PPC::CMPW), CmpReg) .addReg(HiReg) .addReg(ReadAgainReg); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(PPC::PRED_NE) .addReg(CmpReg) .addMBB(readMBB); BB->addSuccessor(readMBB); BB->addSuccessor(sinkMBB); } else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::ADD4); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::ADD4); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I32) BB = EmitAtomicBinary(MI, BB, 4, PPC::ADD4); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I64) BB = EmitAtomicBinary(MI, BB, 8, PPC::ADD8); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::AND); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::AND); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I32) BB = EmitAtomicBinary(MI, BB, 4, PPC::AND); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I64) BB = EmitAtomicBinary(MI, BB, 8, PPC::AND8); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::OR); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::OR); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I32) BB = EmitAtomicBinary(MI, BB, 4, PPC::OR); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I64) BB = EmitAtomicBinary(MI, BB, 8, PPC::OR8); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::XOR); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::XOR); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I32) BB = EmitAtomicBinary(MI, BB, 4, PPC::XOR); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I64) BB = EmitAtomicBinary(MI, BB, 8, PPC::XOR8); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::NAND); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::NAND); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I32) BB = EmitAtomicBinary(MI, BB, 4, PPC::NAND); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I64) BB = EmitAtomicBinary(MI, BB, 8, PPC::NAND8); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::SUBF); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::SUBF); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I32) BB = EmitAtomicBinary(MI, BB, 4, PPC::SUBF); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I64) BB = EmitAtomicBinary(MI, BB, 8, PPC::SUBF8); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPW, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPW, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I32) BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPW, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I64) BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPD, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPW, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPW, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I32) BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPW, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I64) BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPD, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPLW, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPLW, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I32) BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPLW, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I64) BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPLD, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPLW, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPLW, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I32) BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPLW, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I64) BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPLD, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, 0); else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, 0); else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I32) BB = EmitAtomicBinary(MI, BB, 4, 0); else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I64) BB = EmitAtomicBinary(MI, BB, 8, 0); else if (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I32 || MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64 || (Subtarget.hasPartwordAtomics() && MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8) || (Subtarget.hasPartwordAtomics() && MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16)) { bool is64bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64; auto LoadMnemonic = PPC::LDARX; auto StoreMnemonic = PPC::STDCX; switch (MI.getOpcode()) { default: llvm_unreachable("Compare and swap of unknown size"); case PPC::ATOMIC_CMP_SWAP_I8: LoadMnemonic = PPC::LBARX; StoreMnemonic = PPC::STBCX; assert(Subtarget.hasPartwordAtomics() && "No support partword atomics."); break; case PPC::ATOMIC_CMP_SWAP_I16: LoadMnemonic = PPC::LHARX; StoreMnemonic = PPC::STHCX; assert(Subtarget.hasPartwordAtomics() && "No support partword atomics."); break; case PPC::ATOMIC_CMP_SWAP_I32: LoadMnemonic = PPC::LWARX; StoreMnemonic = PPC::STWCX; break; case PPC::ATOMIC_CMP_SWAP_I64: LoadMnemonic = PPC::LDARX; StoreMnemonic = PPC::STDCX; break; } unsigned dest = MI.getOperand(0).getReg(); unsigned ptrA = MI.getOperand(1).getReg(); unsigned ptrB = MI.getOperand(2).getReg(); unsigned oldval = MI.getOperand(3).getReg(); unsigned newval = MI.getOperand(4).getReg(); DebugLoc dl = MI.getDebugLoc(); MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); F->insert(It, loop1MBB); F->insert(It, loop2MBB); F->insert(It, midMBB); F->insert(It, exitMBB); exitMBB->splice(exitMBB->begin(), BB, std::next(MachineBasicBlock::iterator(MI)), BB->end()); exitMBB->transferSuccessorsAndUpdatePHIs(BB); // thisMBB: // ... // fallthrough --> loopMBB BB->addSuccessor(loop1MBB); // loop1MBB: // l[bhwd]arx dest, ptr // cmp[wd] dest, oldval // bne- midMBB // loop2MBB: // st[bhwd]cx. newval, ptr // bne- loopMBB // b exitBB // midMBB: // st[bhwd]cx. dest, ptr // exitBB: BB = loop1MBB; BuildMI(BB, dl, TII->get(LoadMnemonic), dest).addReg(ptrA).addReg(ptrB); BuildMI(BB, dl, TII->get(is64bit ? PPC::CMPD : PPC::CMPW), PPC::CR0) .addReg(oldval) .addReg(dest); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(PPC::PRED_NE) .addReg(PPC::CR0) .addMBB(midMBB); BB->addSuccessor(loop2MBB); BB->addSuccessor(midMBB); BB = loop2MBB; BuildMI(BB, dl, TII->get(StoreMnemonic)) .addReg(newval) .addReg(ptrA) .addReg(ptrB); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(PPC::PRED_NE) .addReg(PPC::CR0) .addMBB(loop1MBB); BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB); BB->addSuccessor(loop1MBB); BB->addSuccessor(exitMBB); BB = midMBB; BuildMI(BB, dl, TII->get(StoreMnemonic)) .addReg(dest) .addReg(ptrA) .addReg(ptrB); BB->addSuccessor(exitMBB); // exitMBB: // ... BB = exitMBB; } else if (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8 || MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16) { // We must use 64-bit registers for addresses when targeting 64-bit, // since we're actually doing arithmetic on them. Other registers // can be 32-bit. bool is64bit = Subtarget.isPPC64(); bool isLittleEndian = Subtarget.isLittleEndian(); bool is8bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8; unsigned dest = MI.getOperand(0).getReg(); unsigned ptrA = MI.getOperand(1).getReg(); unsigned ptrB = MI.getOperand(2).getReg(); unsigned oldval = MI.getOperand(3).getReg(); unsigned newval = MI.getOperand(4).getReg(); DebugLoc dl = MI.getDebugLoc(); MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); F->insert(It, loop1MBB); F->insert(It, loop2MBB); F->insert(It, midMBB); F->insert(It, exitMBB); exitMBB->splice(exitMBB->begin(), BB, std::next(MachineBasicBlock::iterator(MI)), BB->end()); exitMBB->transferSuccessorsAndUpdatePHIs(BB); MachineRegisterInfo &RegInfo = F->getRegInfo(); const TargetRegisterClass *RC = is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; const TargetRegisterClass *GPRC = &PPC::GPRCRegClass; Register PtrReg = RegInfo.createVirtualRegister(RC); Register Shift1Reg = RegInfo.createVirtualRegister(GPRC); Register ShiftReg = isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(GPRC); Register NewVal2Reg = RegInfo.createVirtualRegister(GPRC); Register NewVal3Reg = RegInfo.createVirtualRegister(GPRC); Register OldVal2Reg = RegInfo.createVirtualRegister(GPRC); Register OldVal3Reg = RegInfo.createVirtualRegister(GPRC); Register MaskReg = RegInfo.createVirtualRegister(GPRC); Register Mask2Reg = RegInfo.createVirtualRegister(GPRC); Register Mask3Reg = RegInfo.createVirtualRegister(GPRC); Register Tmp2Reg = RegInfo.createVirtualRegister(GPRC); Register Tmp4Reg = RegInfo.createVirtualRegister(GPRC); Register TmpDestReg = RegInfo.createVirtualRegister(GPRC); Register Ptr1Reg; Register TmpReg = RegInfo.createVirtualRegister(GPRC); Register ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO; // thisMBB: // ... // fallthrough --> loopMBB BB->addSuccessor(loop1MBB); // The 4-byte load must be aligned, while a char or short may be // anywhere in the word. Hence all this nasty bookkeeping code. // add ptr1, ptrA, ptrB [copy if ptrA==0] // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27] // xori shift, shift1, 24 [16] // rlwinm ptr, ptr1, 0, 0, 29 // slw newval2, newval, shift // slw oldval2, oldval,shift // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535] // slw mask, mask2, shift // and newval3, newval2, mask // and oldval3, oldval2, mask // loop1MBB: // lwarx tmpDest, ptr // and tmp, tmpDest, mask // cmpw tmp, oldval3 // bne- midMBB // loop2MBB: // andc tmp2, tmpDest, mask // or tmp4, tmp2, newval3 // stwcx. tmp4, ptr // bne- loop1MBB // b exitBB // midMBB: // stwcx. tmpDest, ptr // exitBB: // srw dest, tmpDest, shift if (ptrA != ZeroReg) { Ptr1Reg = RegInfo.createVirtualRegister(RC); BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg) .addReg(ptrA) .addReg(ptrB); } else { Ptr1Reg = ptrB; } // We need use 32-bit subregister to avoid mismatch register class in 64-bit // mode. BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg) .addReg(Ptr1Reg, 0, is64bit ? PPC::sub_32 : 0) .addImm(3) .addImm(27) .addImm(is8bit ? 28 : 27); if (!isLittleEndian) BuildMI(BB, dl, TII->get(PPC::XORI), ShiftReg) .addReg(Shift1Reg) .addImm(is8bit ? 24 : 16); if (is64bit) BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg) .addReg(Ptr1Reg) .addImm(0) .addImm(61); else BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg) .addReg(Ptr1Reg) .addImm(0) .addImm(0) .addImm(29); BuildMI(BB, dl, TII->get(PPC::SLW), NewVal2Reg) .addReg(newval) .addReg(ShiftReg); BuildMI(BB, dl, TII->get(PPC::SLW), OldVal2Reg) .addReg(oldval) .addReg(ShiftReg); if (is8bit) BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255); else { BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0); BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg) .addReg(Mask3Reg) .addImm(65535); } BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg) .addReg(Mask2Reg) .addReg(ShiftReg); BuildMI(BB, dl, TII->get(PPC::AND), NewVal3Reg) .addReg(NewVal2Reg) .addReg(MaskReg); BuildMI(BB, dl, TII->get(PPC::AND), OldVal3Reg) .addReg(OldVal2Reg) .addReg(MaskReg); BB = loop1MBB; BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg) .addReg(ZeroReg) .addReg(PtrReg); BuildMI(BB, dl, TII->get(PPC::AND), TmpReg) .addReg(TmpDestReg) .addReg(MaskReg); BuildMI(BB, dl, TII->get(PPC::CMPW), PPC::CR0) .addReg(TmpReg) .addReg(OldVal3Reg); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(PPC::PRED_NE) .addReg(PPC::CR0) .addMBB(midMBB); BB->addSuccessor(loop2MBB); BB->addSuccessor(midMBB); BB = loop2MBB; BuildMI(BB, dl, TII->get(PPC::ANDC), Tmp2Reg) .addReg(TmpDestReg) .addReg(MaskReg); BuildMI(BB, dl, TII->get(PPC::OR), Tmp4Reg) .addReg(Tmp2Reg) .addReg(NewVal3Reg); BuildMI(BB, dl, TII->get(PPC::STWCX)) .addReg(Tmp4Reg) .addReg(ZeroReg) .addReg(PtrReg); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(PPC::PRED_NE) .addReg(PPC::CR0) .addMBB(loop1MBB); BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB); BB->addSuccessor(loop1MBB); BB->addSuccessor(exitMBB); BB = midMBB; BuildMI(BB, dl, TII->get(PPC::STWCX)) .addReg(TmpDestReg) .addReg(ZeroReg) .addReg(PtrReg); BB->addSuccessor(exitMBB); // exitMBB: // ... BB = exitMBB; BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest) .addReg(TmpReg) .addReg(ShiftReg); } else if (MI.getOpcode() == PPC::FADDrtz) { // This pseudo performs an FADD with rounding mode temporarily forced // to round-to-zero. We emit this via custom inserter since the FPSCR // is not modeled at the SelectionDAG level. unsigned Dest = MI.getOperand(0).getReg(); unsigned Src1 = MI.getOperand(1).getReg(); unsigned Src2 = MI.getOperand(2).getReg(); DebugLoc dl = MI.getDebugLoc(); MachineRegisterInfo &RegInfo = F->getRegInfo(); unsigned MFFSReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass); // Save FPSCR value. BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), MFFSReg); // Set rounding mode to round-to-zero. BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB1)).addImm(31); BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB0)).addImm(30); // Perform addition. BuildMI(*BB, MI, dl, TII->get(PPC::FADD), Dest).addReg(Src1).addReg(Src2); // Restore FPSCR value. BuildMI(*BB, MI, dl, TII->get(PPC::MTFSFb)).addImm(1).addReg(MFFSReg); } else if (MI.getOpcode() == PPC::ANDIo_1_EQ_BIT || MI.getOpcode() == PPC::ANDIo_1_GT_BIT || MI.getOpcode() == PPC::ANDIo_1_EQ_BIT8 || MI.getOpcode() == PPC::ANDIo_1_GT_BIT8) { unsigned Opcode = (MI.getOpcode() == PPC::ANDIo_1_EQ_BIT8 || MI.getOpcode() == PPC::ANDIo_1_GT_BIT8) ? PPC::ANDIo8 : PPC::ANDIo; bool isEQ = (MI.getOpcode() == PPC::ANDIo_1_EQ_BIT || MI.getOpcode() == PPC::ANDIo_1_EQ_BIT8); MachineRegisterInfo &RegInfo = F->getRegInfo(); unsigned Dest = RegInfo.createVirtualRegister( Opcode == PPC::ANDIo ? &PPC::GPRCRegClass : &PPC::G8RCRegClass); DebugLoc dl = MI.getDebugLoc(); BuildMI(*BB, MI, dl, TII->get(Opcode), Dest) .addReg(MI.getOperand(1).getReg()) .addImm(1); BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), MI.getOperand(0).getReg()) .addReg(isEQ ? PPC::CR0EQ : PPC::CR0GT); } else if (MI.getOpcode() == PPC::TCHECK_RET) { DebugLoc Dl = MI.getDebugLoc(); MachineRegisterInfo &RegInfo = F->getRegInfo(); unsigned CRReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass); BuildMI(*BB, MI, Dl, TII->get(PPC::TCHECK), CRReg); BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY), MI.getOperand(0).getReg()) .addReg(CRReg); } else if (MI.getOpcode() == PPC::TBEGIN_RET) { DebugLoc Dl = MI.getDebugLoc(); unsigned Imm = MI.getOperand(1).getImm(); BuildMI(*BB, MI, Dl, TII->get(PPC::TBEGIN)).addImm(Imm); BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY), MI.getOperand(0).getReg()) .addReg(PPC::CR0EQ); } else if (MI.getOpcode() == PPC::SETRNDi) { DebugLoc dl = MI.getDebugLoc(); unsigned OldFPSCRReg = MI.getOperand(0).getReg(); // Save FPSCR value. BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), OldFPSCRReg); // The floating point rounding mode is in the bits 62:63 of FPCSR, and has // the following settings: // 00 Round to nearest // 01 Round to 0 // 10 Round to +inf // 11 Round to -inf // When the operand is immediate, using the two least significant bits of // the immediate to set the bits 62:63 of FPSCR. unsigned Mode = MI.getOperand(1).getImm(); BuildMI(*BB, MI, dl, TII->get((Mode & 1) ? PPC::MTFSB1 : PPC::MTFSB0)) .addImm(31); BuildMI(*BB, MI, dl, TII->get((Mode & 2) ? PPC::MTFSB1 : PPC::MTFSB0)) .addImm(30); } else if (MI.getOpcode() == PPC::SETRND) { DebugLoc dl = MI.getDebugLoc(); // Copy register from F8RCRegClass::SrcReg to G8RCRegClass::DestReg // or copy register from G8RCRegClass::SrcReg to F8RCRegClass::DestReg. // If the target doesn't have DirectMove, we should use stack to do the // conversion, because the target doesn't have the instructions like mtvsrd // or mfvsrd to do this conversion directly. auto copyRegFromG8RCOrF8RC = [&] (unsigned DestReg, unsigned SrcReg) { if (Subtarget.hasDirectMove()) { BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), DestReg) .addReg(SrcReg); } else { // Use stack to do the register copy. unsigned StoreOp = PPC::STD, LoadOp = PPC::LFD; MachineRegisterInfo &RegInfo = F->getRegInfo(); const TargetRegisterClass *RC = RegInfo.getRegClass(SrcReg); if (RC == &PPC::F8RCRegClass) { // Copy register from F8RCRegClass to G8RCRegclass. assert((RegInfo.getRegClass(DestReg) == &PPC::G8RCRegClass) && "Unsupported RegClass."); StoreOp = PPC::STFD; LoadOp = PPC::LD; } else { // Copy register from G8RCRegClass to F8RCRegclass. assert((RegInfo.getRegClass(SrcReg) == &PPC::G8RCRegClass) && (RegInfo.getRegClass(DestReg) == &PPC::F8RCRegClass) && "Unsupported RegClass."); } MachineFrameInfo &MFI = F->getFrameInfo(); int FrameIdx = MFI.CreateStackObject(8, 8, false); MachineMemOperand *MMOStore = F->getMachineMemOperand( MachinePointerInfo::getFixedStack(*F, FrameIdx, 0), MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx), MFI.getObjectAlignment(FrameIdx)); // Store the SrcReg into the stack. BuildMI(*BB, MI, dl, TII->get(StoreOp)) .addReg(SrcReg) .addImm(0) .addFrameIndex(FrameIdx) .addMemOperand(MMOStore); MachineMemOperand *MMOLoad = F->getMachineMemOperand( MachinePointerInfo::getFixedStack(*F, FrameIdx, 0), MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx), MFI.getObjectAlignment(FrameIdx)); // Load from the stack where SrcReg is stored, and save to DestReg, // so we have done the RegClass conversion from RegClass::SrcReg to // RegClass::DestReg. BuildMI(*BB, MI, dl, TII->get(LoadOp), DestReg) .addImm(0) .addFrameIndex(FrameIdx) .addMemOperand(MMOLoad); } }; unsigned OldFPSCRReg = MI.getOperand(0).getReg(); // Save FPSCR value. BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), OldFPSCRReg); // When the operand is gprc register, use two least significant bits of the // register and mtfsf instruction to set the bits 62:63 of FPSCR. // // copy OldFPSCRTmpReg, OldFPSCRReg // (INSERT_SUBREG ExtSrcReg, (IMPLICIT_DEF ImDefReg), SrcOp, 1) // rldimi NewFPSCRTmpReg, ExtSrcReg, OldFPSCRReg, 0, 62 // copy NewFPSCRReg, NewFPSCRTmpReg // mtfsf 255, NewFPSCRReg MachineOperand SrcOp = MI.getOperand(1); MachineRegisterInfo &RegInfo = F->getRegInfo(); unsigned OldFPSCRTmpReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); copyRegFromG8RCOrF8RC(OldFPSCRTmpReg, OldFPSCRReg); unsigned ImDefReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); unsigned ExtSrcReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); // The first operand of INSERT_SUBREG should be a register which has // subregisters, we only care about its RegClass, so we should use an // IMPLICIT_DEF register. BuildMI(*BB, MI, dl, TII->get(TargetOpcode::IMPLICIT_DEF), ImDefReg); BuildMI(*BB, MI, dl, TII->get(PPC::INSERT_SUBREG), ExtSrcReg) .addReg(ImDefReg) .add(SrcOp) .addImm(1); unsigned NewFPSCRTmpReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); BuildMI(*BB, MI, dl, TII->get(PPC::RLDIMI), NewFPSCRTmpReg) .addReg(OldFPSCRTmpReg) .addReg(ExtSrcReg) .addImm(0) .addImm(62); unsigned NewFPSCRReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass); copyRegFromG8RCOrF8RC(NewFPSCRReg, NewFPSCRTmpReg); // The mask 255 means that put the 32:63 bits of NewFPSCRReg to the 32:63 // bits of FPSCR. BuildMI(*BB, MI, dl, TII->get(PPC::MTFSF)) .addImm(255) .addReg(NewFPSCRReg) .addImm(0) .addImm(0); } else { llvm_unreachable("Unexpected instr type to insert"); } MI.eraseFromParent(); // The pseudo instruction is gone now. return BB; } //===----------------------------------------------------------------------===// // Target Optimization Hooks //===----------------------------------------------------------------------===// static int getEstimateRefinementSteps(EVT VT, const PPCSubtarget &Subtarget) { // For the estimates, convergence is quadratic, so we essentially double the // number of digits correct after every iteration. For both FRE and FRSQRTE, // the minimum architected relative accuracy is 2^-5. When hasRecipPrec(), // this is 2^-14. IEEE float has 23 digits and double has 52 digits. int RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3; if (VT.getScalarType() == MVT::f64) RefinementSteps++; return RefinementSteps; } SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const { EVT VT = Operand.getValueType(); if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) || (VT == MVT::f64 && Subtarget.hasFRSQRTE()) || (VT == MVT::v4f32 && Subtarget.hasAltivec()) || (VT == MVT::v2f64 && Subtarget.hasVSX()) || (VT == MVT::v4f32 && Subtarget.hasQPX()) || (VT == MVT::v4f64 && Subtarget.hasQPX())) { if (RefinementSteps == ReciprocalEstimate::Unspecified) RefinementSteps = getEstimateRefinementSteps(VT, Subtarget); // The Newton-Raphson computation with a single constant does not provide // enough accuracy on some CPUs. UseOneConstNR = !Subtarget.needsTwoConstNR(); return DAG.getNode(PPCISD::FRSQRTE, SDLoc(Operand), VT, Operand); } return SDValue(); } SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const { EVT VT = Operand.getValueType(); if ((VT == MVT::f32 && Subtarget.hasFRES()) || (VT == MVT::f64 && Subtarget.hasFRE()) || (VT == MVT::v4f32 && Subtarget.hasAltivec()) || (VT == MVT::v2f64 && Subtarget.hasVSX()) || (VT == MVT::v4f32 && Subtarget.hasQPX()) || (VT == MVT::v4f64 && Subtarget.hasQPX())) { if (RefinementSteps == ReciprocalEstimate::Unspecified) RefinementSteps = getEstimateRefinementSteps(VT, Subtarget); return DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand); } return SDValue(); } unsigned PPCTargetLowering::combineRepeatedFPDivisors() const { // Note: This functionality is used only when unsafe-fp-math is enabled, and // on cores with reciprocal estimates (which are used when unsafe-fp-math is // enabled for division), this functionality is redundant with the default // combiner logic (once the division -> reciprocal/multiply transformation // has taken place). As a result, this matters more for older cores than for // newer ones. // Combine multiple FDIVs with the same divisor into multiple FMULs by the // reciprocal if there are two or more FDIVs (for embedded cores with only // one FP pipeline) for three or more FDIVs (for generic OOO cores). switch (Subtarget.getDarwinDirective()) { default: return 3; case PPC::DIR_440: case PPC::DIR_A2: case PPC::DIR_E500: case PPC::DIR_E500mc: case PPC::DIR_E5500: return 2; } } // isConsecutiveLSLoc needs to work even if all adds have not yet been // collapsed, and so we need to look through chains of them. static void getBaseWithConstantOffset(SDValue Loc, SDValue &Base, int64_t& Offset, SelectionDAG &DAG) { if (DAG.isBaseWithConstantOffset(Loc)) { Base = Loc.getOperand(0); Offset += cast(Loc.getOperand(1))->getSExtValue(); // The base might itself be a base plus an offset, and if so, accumulate // that as well. getBaseWithConstantOffset(Loc.getOperand(0), Base, Offset, DAG); } } static bool isConsecutiveLSLoc(SDValue Loc, EVT VT, LSBaseSDNode *Base, unsigned Bytes, int Dist, SelectionDAG &DAG) { if (VT.getSizeInBits() / 8 != Bytes) return false; SDValue BaseLoc = Base->getBasePtr(); if (Loc.getOpcode() == ISD::FrameIndex) { if (BaseLoc.getOpcode() != ISD::FrameIndex) return false; const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); int FI = cast(Loc)->getIndex(); int BFI = cast(BaseLoc)->getIndex(); int FS = MFI.getObjectSize(FI); int BFS = MFI.getObjectSize(BFI); if (FS != BFS || FS != (int)Bytes) return false; return MFI.getObjectOffset(FI) == (MFI.getObjectOffset(BFI) + Dist*Bytes); } SDValue Base1 = Loc, Base2 = BaseLoc; int64_t Offset1 = 0, Offset2 = 0; getBaseWithConstantOffset(Loc, Base1, Offset1, DAG); getBaseWithConstantOffset(BaseLoc, Base2, Offset2, DAG); if (Base1 == Base2 && Offset1 == (Offset2 + Dist * Bytes)) return true; const TargetLowering &TLI = DAG.getTargetLoweringInfo(); const GlobalValue *GV1 = nullptr; const GlobalValue *GV2 = nullptr; Offset1 = 0; Offset2 = 0; bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1); bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); if (isGA1 && isGA2 && GV1 == GV2) return Offset1 == (Offset2 + Dist*Bytes); return false; } // Like SelectionDAG::isConsecutiveLoad, but also works for stores, and does // not enforce equality of the chain operands. static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base, unsigned Bytes, int Dist, SelectionDAG &DAG) { if (LSBaseSDNode *LS = dyn_cast(N)) { EVT VT = LS->getMemoryVT(); SDValue Loc = LS->getBasePtr(); return isConsecutiveLSLoc(Loc, VT, Base, Bytes, Dist, DAG); } if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) { EVT VT; switch (cast(N->getOperand(1))->getZExtValue()) { default: return false; case Intrinsic::ppc_qpx_qvlfd: case Intrinsic::ppc_qpx_qvlfda: VT = MVT::v4f64; break; case Intrinsic::ppc_qpx_qvlfs: case Intrinsic::ppc_qpx_qvlfsa: VT = MVT::v4f32; break; case Intrinsic::ppc_qpx_qvlfcd: case Intrinsic::ppc_qpx_qvlfcda: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvlfcs: case Intrinsic::ppc_qpx_qvlfcsa: VT = MVT::v2f32; break; case Intrinsic::ppc_qpx_qvlfiwa: case Intrinsic::ppc_qpx_qvlfiwz: case Intrinsic::ppc_altivec_lvx: case Intrinsic::ppc_altivec_lvxl: case Intrinsic::ppc_vsx_lxvw4x: case Intrinsic::ppc_vsx_lxvw4x_be: VT = MVT::v4i32; break; case Intrinsic::ppc_vsx_lxvd2x: case Intrinsic::ppc_vsx_lxvd2x_be: VT = MVT::v2f64; break; case Intrinsic::ppc_altivec_lvebx: VT = MVT::i8; break; case Intrinsic::ppc_altivec_lvehx: VT = MVT::i16; break; case Intrinsic::ppc_altivec_lvewx: VT = MVT::i32; break; } return isConsecutiveLSLoc(N->getOperand(2), VT, Base, Bytes, Dist, DAG); } if (N->getOpcode() == ISD::INTRINSIC_VOID) { EVT VT; switch (cast(N->getOperand(1))->getZExtValue()) { default: return false; case Intrinsic::ppc_qpx_qvstfd: case Intrinsic::ppc_qpx_qvstfda: VT = MVT::v4f64; break; case Intrinsic::ppc_qpx_qvstfs: case Intrinsic::ppc_qpx_qvstfsa: VT = MVT::v4f32; break; case Intrinsic::ppc_qpx_qvstfcd: case Intrinsic::ppc_qpx_qvstfcda: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvstfcs: case Intrinsic::ppc_qpx_qvstfcsa: VT = MVT::v2f32; break; case Intrinsic::ppc_qpx_qvstfiw: case Intrinsic::ppc_qpx_qvstfiwa: case Intrinsic::ppc_altivec_stvx: case Intrinsic::ppc_altivec_stvxl: case Intrinsic::ppc_vsx_stxvw4x: VT = MVT::v4i32; break; case Intrinsic::ppc_vsx_stxvd2x: VT = MVT::v2f64; break; case Intrinsic::ppc_vsx_stxvw4x_be: VT = MVT::v4i32; break; case Intrinsic::ppc_vsx_stxvd2x_be: VT = MVT::v2f64; break; case Intrinsic::ppc_altivec_stvebx: VT = MVT::i8; break; case Intrinsic::ppc_altivec_stvehx: VT = MVT::i16; break; case Intrinsic::ppc_altivec_stvewx: VT = MVT::i32; break; } return isConsecutiveLSLoc(N->getOperand(3), VT, Base, Bytes, Dist, DAG); } return false; } // Return true is there is a nearyby consecutive load to the one provided // (regardless of alignment). We search up and down the chain, looking though // token factors and other loads (but nothing else). As a result, a true result // indicates that it is safe to create a new consecutive load adjacent to the // load provided. static bool findConsecutiveLoad(LoadSDNode *LD, SelectionDAG &DAG) { SDValue Chain = LD->getChain(); EVT VT = LD->getMemoryVT(); SmallSet LoadRoots; SmallVector Queue(1, Chain.getNode()); SmallSet Visited; // First, search up the chain, branching to follow all token-factor operands. // If we find a consecutive load, then we're done, otherwise, record all // nodes just above the top-level loads and token factors. while (!Queue.empty()) { SDNode *ChainNext = Queue.pop_back_val(); if (!Visited.insert(ChainNext).second) continue; if (MemSDNode *ChainLD = dyn_cast(ChainNext)) { if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG)) return true; if (!Visited.count(ChainLD->getChain().getNode())) Queue.push_back(ChainLD->getChain().getNode()); } else if (ChainNext->getOpcode() == ISD::TokenFactor) { for (const SDUse &O : ChainNext->ops()) if (!Visited.count(O.getNode())) Queue.push_back(O.getNode()); } else LoadRoots.insert(ChainNext); } // Second, search down the chain, starting from the top-level nodes recorded // in the first phase. These top-level nodes are the nodes just above all // loads and token factors. Starting with their uses, recursively look though // all loads (just the chain uses) and token factors to find a consecutive // load. Visited.clear(); Queue.clear(); for (SmallSet::iterator I = LoadRoots.begin(), IE = LoadRoots.end(); I != IE; ++I) { Queue.push_back(*I); while (!Queue.empty()) { SDNode *LoadRoot = Queue.pop_back_val(); if (!Visited.insert(LoadRoot).second) continue; if (MemSDNode *ChainLD = dyn_cast(LoadRoot)) if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG)) return true; for (SDNode::use_iterator UI = LoadRoot->use_begin(), UE = LoadRoot->use_end(); UI != UE; ++UI) if (((isa(*UI) && cast(*UI)->getChain().getNode() == LoadRoot) || UI->getOpcode() == ISD::TokenFactor) && !Visited.count(*UI)) Queue.push_back(*UI); } } return false; } /// This function is called when we have proved that a SETCC node can be replaced /// by subtraction (and other supporting instructions) so that the result of /// comparison is kept in a GPR instead of CR. This function is purely for /// codegen purposes and has some flags to guide the codegen process. static SDValue generateEquivalentSub(SDNode *N, int Size, bool Complement, bool Swap, SDLoc &DL, SelectionDAG &DAG) { assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected."); // Zero extend the operands to the largest legal integer. Originally, they // must be of a strictly smaller size. auto Op0 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(0), DAG.getConstant(Size, DL, MVT::i32)); auto Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1), DAG.getConstant(Size, DL, MVT::i32)); // Swap if needed. Depends on the condition code. if (Swap) std::swap(Op0, Op1); // Subtract extended integers. auto SubNode = DAG.getNode(ISD::SUB, DL, MVT::i64, Op0, Op1); // Move the sign bit to the least significant position and zero out the rest. // Now the least significant bit carries the result of original comparison. auto Shifted = DAG.getNode(ISD::SRL, DL, MVT::i64, SubNode, DAG.getConstant(Size - 1, DL, MVT::i32)); auto Final = Shifted; // Complement the result if needed. Based on the condition code. if (Complement) Final = DAG.getNode(ISD::XOR, DL, MVT::i64, Shifted, DAG.getConstant(1, DL, MVT::i64)); return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Final); } SDValue PPCTargetLowering::ConvertSETCCToSubtract(SDNode *N, DAGCombinerInfo &DCI) const { assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected."); SelectionDAG &DAG = DCI.DAG; SDLoc DL(N); // Size of integers being compared has a critical role in the following // analysis, so we prefer to do this when all types are legal. if (!DCI.isAfterLegalizeDAG()) return SDValue(); // If all users of SETCC extend its value to a legal integer type // then we replace SETCC with a subtraction for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); UI != UE; ++UI) { if (UI->getOpcode() != ISD::ZERO_EXTEND) return SDValue(); } ISD::CondCode CC = cast(N->getOperand(2))->get(); auto OpSize = N->getOperand(0).getValueSizeInBits(); unsigned Size = DAG.getDataLayout().getLargestLegalIntTypeSizeInBits(); if (OpSize < Size) { switch (CC) { default: break; case ISD::SETULT: return generateEquivalentSub(N, Size, false, false, DL, DAG); case ISD::SETULE: return generateEquivalentSub(N, Size, true, true, DL, DAG); case ISD::SETUGT: return generateEquivalentSub(N, Size, false, true, DL, DAG); case ISD::SETUGE: return generateEquivalentSub(N, Size, true, false, DL, DAG); } } return SDValue(); } SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); assert(Subtarget.useCRBits() && "Expecting to be tracking CR bits"); // If we're tracking CR bits, we need to be careful that we don't have: // trunc(binary-ops(zext(x), zext(y))) // or // trunc(binary-ops(binary-ops(zext(x), zext(y)), ...) // such that we're unnecessarily moving things into GPRs when it would be // better to keep them in CR bits. // Note that trunc here can be an actual i1 trunc, or can be the effective // truncation that comes from a setcc or select_cc. if (N->getOpcode() == ISD::TRUNCATE && N->getValueType(0) != MVT::i1) return SDValue(); if (N->getOperand(0).getValueType() != MVT::i32 && N->getOperand(0).getValueType() != MVT::i64) return SDValue(); if (N->getOpcode() == ISD::SETCC || N->getOpcode() == ISD::SELECT_CC) { // If we're looking at a comparison, then we need to make sure that the // high bits (all except for the first) don't matter the result. ISD::CondCode CC = cast(N->getOperand( N->getOpcode() == ISD::SETCC ? 2 : 4))->get(); unsigned OpBits = N->getOperand(0).getValueSizeInBits(); if (ISD::isSignedIntSetCC(CC)) { if (DAG.ComputeNumSignBits(N->getOperand(0)) != OpBits || DAG.ComputeNumSignBits(N->getOperand(1)) != OpBits) return SDValue(); } else if (ISD::isUnsignedIntSetCC(CC)) { if (!DAG.MaskedValueIsZero(N->getOperand(0), APInt::getHighBitsSet(OpBits, OpBits-1)) || !DAG.MaskedValueIsZero(N->getOperand(1), APInt::getHighBitsSet(OpBits, OpBits-1))) return (N->getOpcode() == ISD::SETCC ? ConvertSETCCToSubtract(N, DCI) : SDValue()); } else { // This is neither a signed nor an unsigned comparison, just make sure // that the high bits are equal. KnownBits Op1Known = DAG.computeKnownBits(N->getOperand(0)); KnownBits Op2Known = DAG.computeKnownBits(N->getOperand(1)); // We don't really care about what is known about the first bit (if // anything), so clear it in all masks prior to comparing them. Op1Known.Zero.clearBit(0); Op1Known.One.clearBit(0); Op2Known.Zero.clearBit(0); Op2Known.One.clearBit(0); if (Op1Known.Zero != Op2Known.Zero || Op1Known.One != Op2Known.One) return SDValue(); } } // We now know that the higher-order bits are irrelevant, we just need to // make sure that all of the intermediate operations are bit operations, and // all inputs are extensions. if (N->getOperand(0).getOpcode() != ISD::AND && N->getOperand(0).getOpcode() != ISD::OR && N->getOperand(0).getOpcode() != ISD::XOR && N->getOperand(0).getOpcode() != ISD::SELECT && N->getOperand(0).getOpcode() != ISD::SELECT_CC && N->getOperand(0).getOpcode() != ISD::TRUNCATE && N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND && N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND && N->getOperand(0).getOpcode() != ISD::ANY_EXTEND) return SDValue(); if ((N->getOpcode() == ISD::SETCC || N->getOpcode() == ISD::SELECT_CC) && N->getOperand(1).getOpcode() != ISD::AND && N->getOperand(1).getOpcode() != ISD::OR && N->getOperand(1).getOpcode() != ISD::XOR && N->getOperand(1).getOpcode() != ISD::SELECT && N->getOperand(1).getOpcode() != ISD::SELECT_CC && N->getOperand(1).getOpcode() != ISD::TRUNCATE && N->getOperand(1).getOpcode() != ISD::SIGN_EXTEND && N->getOperand(1).getOpcode() != ISD::ZERO_EXTEND && N->getOperand(1).getOpcode() != ISD::ANY_EXTEND) return SDValue(); SmallVector Inputs; SmallVector BinOps, PromOps; SmallPtrSet Visited; for (unsigned i = 0; i < 2; ++i) { if (((N->getOperand(i).getOpcode() == ISD::SIGN_EXTEND || N->getOperand(i).getOpcode() == ISD::ZERO_EXTEND || N->getOperand(i).getOpcode() == ISD::ANY_EXTEND) && N->getOperand(i).getOperand(0).getValueType() == MVT::i1) || isa(N->getOperand(i))) Inputs.push_back(N->getOperand(i)); else BinOps.push_back(N->getOperand(i)); if (N->getOpcode() == ISD::TRUNCATE) break; } // Visit all inputs, collect all binary operations (and, or, xor and // select) that are all fed by extensions. while (!BinOps.empty()) { SDValue BinOp = BinOps.back(); BinOps.pop_back(); if (!Visited.insert(BinOp.getNode()).second) continue; PromOps.push_back(BinOp); for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) { // The condition of the select is not promoted. if (BinOp.getOpcode() == ISD::SELECT && i == 0) continue; if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3) continue; if (((BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND || BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND || BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) && BinOp.getOperand(i).getOperand(0).getValueType() == MVT::i1) || isa(BinOp.getOperand(i))) { Inputs.push_back(BinOp.getOperand(i)); } else if (BinOp.getOperand(i).getOpcode() == ISD::AND || BinOp.getOperand(i).getOpcode() == ISD::OR || BinOp.getOperand(i).getOpcode() == ISD::XOR || BinOp.getOperand(i).getOpcode() == ISD::SELECT || BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC || BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE || BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND || BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND || BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) { BinOps.push_back(BinOp.getOperand(i)); } else { // We have an input that is not an extension or another binary // operation; we'll abort this transformation. return SDValue(); } } } // Make sure that this is a self-contained cluster of operations (which // is not quite the same thing as saying that everything has only one // use). for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { if (isa(Inputs[i])) continue; for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(), UE = Inputs[i].getNode()->use_end(); UI != UE; ++UI) { SDNode *User = *UI; if (User != N && !Visited.count(User)) return SDValue(); // Make sure that we're not going to promote the non-output-value // operand(s) or SELECT or SELECT_CC. // FIXME: Although we could sometimes handle this, and it does occur in // practice that one of the condition inputs to the select is also one of // the outputs, we currently can't deal with this. if (User->getOpcode() == ISD::SELECT) { if (User->getOperand(0) == Inputs[i]) return SDValue(); } else if (User->getOpcode() == ISD::SELECT_CC) { if (User->getOperand(0) == Inputs[i] || User->getOperand(1) == Inputs[i]) return SDValue(); } } } for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) { for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(), UE = PromOps[i].getNode()->use_end(); UI != UE; ++UI) { SDNode *User = *UI; if (User != N && !Visited.count(User)) return SDValue(); // Make sure that we're not going to promote the non-output-value // operand(s) or SELECT or SELECT_CC. // FIXME: Although we could sometimes handle this, and it does occur in // practice that one of the condition inputs to the select is also one of // the outputs, we currently can't deal with this. if (User->getOpcode() == ISD::SELECT) { if (User->getOperand(0) == PromOps[i]) return SDValue(); } else if (User->getOpcode() == ISD::SELECT_CC) { if (User->getOperand(0) == PromOps[i] || User->getOperand(1) == PromOps[i]) return SDValue(); } } } // Replace all inputs with the extension operand. for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { // Constants may have users outside the cluster of to-be-promoted nodes, // and so we need to replace those as we do the promotions. if (isa(Inputs[i])) continue; else DAG.ReplaceAllUsesOfValueWith(Inputs[i], Inputs[i].getOperand(0)); } std::list PromOpHandles; for (auto &PromOp : PromOps) PromOpHandles.emplace_back(PromOp); // Replace all operations (these are all the same, but have a different // (i1) return type). DAG.getNode will validate that the types of // a binary operator match, so go through the list in reverse so that // we've likely promoted both operands first. Any intermediate truncations or // extensions disappear. while (!PromOpHandles.empty()) { SDValue PromOp = PromOpHandles.back().getValue(); PromOpHandles.pop_back(); if (PromOp.getOpcode() == ISD::TRUNCATE || PromOp.getOpcode() == ISD::SIGN_EXTEND || PromOp.getOpcode() == ISD::ZERO_EXTEND || PromOp.getOpcode() == ISD::ANY_EXTEND) { if (!isa(PromOp.getOperand(0)) && PromOp.getOperand(0).getValueType() != MVT::i1) { // The operand is not yet ready (see comment below). PromOpHandles.emplace_front(PromOp); continue; } SDValue RepValue = PromOp.getOperand(0); if (isa(RepValue)) RepValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, RepValue); DAG.ReplaceAllUsesOfValueWith(PromOp, RepValue); continue; } unsigned C; switch (PromOp.getOpcode()) { default: C = 0; break; case ISD::SELECT: C = 1; break; case ISD::SELECT_CC: C = 2; break; } if ((!isa(PromOp.getOperand(C)) && PromOp.getOperand(C).getValueType() != MVT::i1) || (!isa(PromOp.getOperand(C+1)) && PromOp.getOperand(C+1).getValueType() != MVT::i1)) { // The to-be-promoted operands of this node have not yet been // promoted (this should be rare because we're going through the // list backward, but if one of the operands has several users in // this cluster of to-be-promoted nodes, it is possible). PromOpHandles.emplace_front(PromOp); continue; } SmallVector Ops(PromOp.getNode()->op_begin(), PromOp.getNode()->op_end()); // If there are any constant inputs, make sure they're replaced now. for (unsigned i = 0; i < 2; ++i) if (isa(Ops[C+i])) Ops[C+i] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ops[C+i]); DAG.ReplaceAllUsesOfValueWith(PromOp, DAG.getNode(PromOp.getOpcode(), dl, MVT::i1, Ops)); } // Now we're left with the initial truncation itself. if (N->getOpcode() == ISD::TRUNCATE) return N->getOperand(0); // Otherwise, this is a comparison. The operands to be compared have just // changed type (to i1), but everything else is the same. return SDValue(N, 0); } SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); // If we're tracking CR bits, we need to be careful that we don't have: // zext(binary-ops(trunc(x), trunc(y))) // or // zext(binary-ops(binary-ops(trunc(x), trunc(y)), ...) // such that we're unnecessarily moving things into CR bits that can more // efficiently stay in GPRs. Note that if we're not certain that the high // bits are set as required by the final extension, we still may need to do // some masking to get the proper behavior. // This same functionality is important on PPC64 when dealing with // 32-to-64-bit extensions; these occur often when 32-bit values are used as // the return values of functions. Because it is so similar, it is handled // here as well. if (N->getValueType(0) != MVT::i32 && N->getValueType(0) != MVT::i64) return SDValue(); if (!((N->getOperand(0).getValueType() == MVT::i1 && Subtarget.useCRBits()) || (N->getOperand(0).getValueType() == MVT::i32 && Subtarget.isPPC64()))) return SDValue(); if (N->getOperand(0).getOpcode() != ISD::AND && N->getOperand(0).getOpcode() != ISD::OR && N->getOperand(0).getOpcode() != ISD::XOR && N->getOperand(0).getOpcode() != ISD::SELECT && N->getOperand(0).getOpcode() != ISD::SELECT_CC) return SDValue(); SmallVector Inputs; SmallVector BinOps(1, N->getOperand(0)), PromOps; SmallPtrSet Visited; // Visit all inputs, collect all binary operations (and, or, xor and // select) that are all fed by truncations. while (!BinOps.empty()) { SDValue BinOp = BinOps.back(); BinOps.pop_back(); if (!Visited.insert(BinOp.getNode()).second) continue; PromOps.push_back(BinOp); for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) { // The condition of the select is not promoted. if (BinOp.getOpcode() == ISD::SELECT && i == 0) continue; if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3) continue; if (BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE || isa(BinOp.getOperand(i))) { Inputs.push_back(BinOp.getOperand(i)); } else if (BinOp.getOperand(i).getOpcode() == ISD::AND || BinOp.getOperand(i).getOpcode() == ISD::OR || BinOp.getOperand(i).getOpcode() == ISD::XOR || BinOp.getOperand(i).getOpcode() == ISD::SELECT || BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC) { BinOps.push_back(BinOp.getOperand(i)); } else { // We have an input that is not a truncation or another binary // operation; we'll abort this transformation. return SDValue(); } } } // The operands of a select that must be truncated when the select is // promoted because the operand is actually part of the to-be-promoted set. DenseMap SelectTruncOp[2]; // Make sure that this is a self-contained cluster of operations (which // is not quite the same thing as saying that everything has only one // use). for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { if (isa(Inputs[i])) continue; for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(), UE = Inputs[i].getNode()->use_end(); UI != UE; ++UI) { SDNode *User = *UI; if (User != N && !Visited.count(User)) return SDValue(); // If we're going to promote the non-output-value operand(s) or SELECT or // SELECT_CC, record them for truncation. if (User->getOpcode() == ISD::SELECT) { if (User->getOperand(0) == Inputs[i]) SelectTruncOp[0].insert(std::make_pair(User, User->getOperand(0).getValueType())); } else if (User->getOpcode() == ISD::SELECT_CC) { if (User->getOperand(0) == Inputs[i]) SelectTruncOp[0].insert(std::make_pair(User, User->getOperand(0).getValueType())); if (User->getOperand(1) == Inputs[i]) SelectTruncOp[1].insert(std::make_pair(User, User->getOperand(1).getValueType())); } } } for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) { for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(), UE = PromOps[i].getNode()->use_end(); UI != UE; ++UI) { SDNode *User = *UI; if (User != N && !Visited.count(User)) return SDValue(); // If we're going to promote the non-output-value operand(s) or SELECT or // SELECT_CC, record them for truncation. if (User->getOpcode() == ISD::SELECT) { if (User->getOperand(0) == PromOps[i]) SelectTruncOp[0].insert(std::make_pair(User, User->getOperand(0).getValueType())); } else if (User->getOpcode() == ISD::SELECT_CC) { if (User->getOperand(0) == PromOps[i]) SelectTruncOp[0].insert(std::make_pair(User, User->getOperand(0).getValueType())); if (User->getOperand(1) == PromOps[i]) SelectTruncOp[1].insert(std::make_pair(User, User->getOperand(1).getValueType())); } } } unsigned PromBits = N->getOperand(0).getValueSizeInBits(); bool ReallyNeedsExt = false; if (N->getOpcode() != ISD::ANY_EXTEND) { // If all of the inputs are not already sign/zero extended, then // we'll still need to do that at the end. for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { if (isa(Inputs[i])) continue; unsigned OpBits = Inputs[i].getOperand(0).getValueSizeInBits(); assert(PromBits < OpBits && "Truncation not to a smaller bit count?"); if ((N->getOpcode() == ISD::ZERO_EXTEND && !DAG.MaskedValueIsZero(Inputs[i].getOperand(0), APInt::getHighBitsSet(OpBits, OpBits-PromBits))) || (N->getOpcode() == ISD::SIGN_EXTEND && DAG.ComputeNumSignBits(Inputs[i].getOperand(0)) < (OpBits-(PromBits-1)))) { ReallyNeedsExt = true; break; } } } // Replace all inputs, either with the truncation operand, or a // truncation or extension to the final output type. for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { // Constant inputs need to be replaced with the to-be-promoted nodes that // use them because they might have users outside of the cluster of // promoted nodes. if (isa(Inputs[i])) continue; SDValue InSrc = Inputs[i].getOperand(0); if (Inputs[i].getValueType() == N->getValueType(0)) DAG.ReplaceAllUsesOfValueWith(Inputs[i], InSrc); else if (N->getOpcode() == ISD::SIGN_EXTEND) DAG.ReplaceAllUsesOfValueWith(Inputs[i], DAG.getSExtOrTrunc(InSrc, dl, N->getValueType(0))); else if (N->getOpcode() == ISD::ZERO_EXTEND) DAG.ReplaceAllUsesOfValueWith(Inputs[i], DAG.getZExtOrTrunc(InSrc, dl, N->getValueType(0))); else DAG.ReplaceAllUsesOfValueWith(Inputs[i], DAG.getAnyExtOrTrunc(InSrc, dl, N->getValueType(0))); } std::list PromOpHandles; for (auto &PromOp : PromOps) PromOpHandles.emplace_back(PromOp); // Replace all operations (these are all the same, but have a different // (promoted) return type). DAG.getNode will validate that the types of // a binary operator match, so go through the list in reverse so that // we've likely promoted both operands first. while (!PromOpHandles.empty()) { SDValue PromOp = PromOpHandles.back().getValue(); PromOpHandles.pop_back(); unsigned C; switch (PromOp.getOpcode()) { default: C = 0; break; case ISD::SELECT: C = 1; break; case ISD::SELECT_CC: C = 2; break; } if ((!isa(PromOp.getOperand(C)) && PromOp.getOperand(C).getValueType() != N->getValueType(0)) || (!isa(PromOp.getOperand(C+1)) && PromOp.getOperand(C+1).getValueType() != N->getValueType(0))) { // The to-be-promoted operands of this node have not yet been // promoted (this should be rare because we're going through the // list backward, but if one of the operands has several users in // this cluster of to-be-promoted nodes, it is possible). PromOpHandles.emplace_front(PromOp); continue; } // For SELECT and SELECT_CC nodes, we do a similar check for any // to-be-promoted comparison inputs. if (PromOp.getOpcode() == ISD::SELECT || PromOp.getOpcode() == ISD::SELECT_CC) { if ((SelectTruncOp[0].count(PromOp.getNode()) && PromOp.getOperand(0).getValueType() != N->getValueType(0)) || (SelectTruncOp[1].count(PromOp.getNode()) && PromOp.getOperand(1).getValueType() != N->getValueType(0))) { PromOpHandles.emplace_front(PromOp); continue; } } SmallVector Ops(PromOp.getNode()->op_begin(), PromOp.getNode()->op_end()); // If this node has constant inputs, then they'll need to be promoted here. for (unsigned i = 0; i < 2; ++i) { if (!isa(Ops[C+i])) continue; if (Ops[C+i].getValueType() == N->getValueType(0)) continue; if (N->getOpcode() == ISD::SIGN_EXTEND) Ops[C+i] = DAG.getSExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); else if (N->getOpcode() == ISD::ZERO_EXTEND) Ops[C+i] = DAG.getZExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); else Ops[C+i] = DAG.getAnyExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); } // If we've promoted the comparison inputs of a SELECT or SELECT_CC, // truncate them again to the original value type. if (PromOp.getOpcode() == ISD::SELECT || PromOp.getOpcode() == ISD::SELECT_CC) { auto SI0 = SelectTruncOp[0].find(PromOp.getNode()); if (SI0 != SelectTruncOp[0].end()) Ops[0] = DAG.getNode(ISD::TRUNCATE, dl, SI0->second, Ops[0]); auto SI1 = SelectTruncOp[1].find(PromOp.getNode()); if (SI1 != SelectTruncOp[1].end()) Ops[1] = DAG.getNode(ISD::TRUNCATE, dl, SI1->second, Ops[1]); } DAG.ReplaceAllUsesOfValueWith(PromOp, DAG.getNode(PromOp.getOpcode(), dl, N->getValueType(0), Ops)); } // Now we're left with the initial extension itself. if (!ReallyNeedsExt) return N->getOperand(0); // To zero extend, just mask off everything except for the first bit (in the // i1 case). if (N->getOpcode() == ISD::ZERO_EXTEND) return DAG.getNode(ISD::AND, dl, N->getValueType(0), N->getOperand(0), DAG.getConstant(APInt::getLowBitsSet( N->getValueSizeInBits(0), PromBits), dl, N->getValueType(0))); assert(N->getOpcode() == ISD::SIGN_EXTEND && "Invalid extension type"); EVT ShiftAmountTy = getShiftAmountTy(N->getValueType(0), DAG.getDataLayout()); SDValue ShiftCst = DAG.getConstant(N->getValueSizeInBits(0) - PromBits, dl, ShiftAmountTy); return DAG.getNode( ISD::SRA, dl, N->getValueType(0), DAG.getNode(ISD::SHL, dl, N->getValueType(0), N->getOperand(0), ShiftCst), ShiftCst); } SDValue PPCTargetLowering::combineSetCC(SDNode *N, DAGCombinerInfo &DCI) const { assert(N->getOpcode() == ISD::SETCC && "Should be called with a SETCC node"); ISD::CondCode CC = cast(N->getOperand(2))->get(); if (CC == ISD::SETNE || CC == ISD::SETEQ) { SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); // If there is a '0 - y' pattern, canonicalize the pattern to the RHS. if (LHS.getOpcode() == ISD::SUB && isNullConstant(LHS.getOperand(0)) && LHS.hasOneUse()) std::swap(LHS, RHS); // x == 0-y --> x+y == 0 // x != 0-y --> x+y != 0 if (RHS.getOpcode() == ISD::SUB && isNullConstant(RHS.getOperand(0)) && RHS.hasOneUse()) { SDLoc DL(N); SelectionDAG &DAG = DCI.DAG; EVT VT = N->getValueType(0); EVT OpVT = LHS.getValueType(); SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LHS, RHS.getOperand(1)); return DAG.getSetCC(DL, VT, Add, DAG.getConstant(0, DL, OpVT), CC); } } return DAGCombineTruncBoolExt(N, DCI); } // Is this an extending load from an f32 to an f64? static bool isFPExtLoad(SDValue Op) { if (LoadSDNode *LD = dyn_cast(Op.getNode())) return LD->getExtensionType() == ISD::EXTLOAD && Op.getValueType() == MVT::f64; return false; } /// Reduces the number of fp-to-int conversion when building a vector. /// /// If this vector is built out of floating to integer conversions, /// transform it to a vector built out of floating point values followed by a /// single floating to integer conversion of the vector. /// Namely (build_vector (fptosi $A), (fptosi $B), ...) /// becomes (fptosi (build_vector ($A, $B, ...))) SDValue PPCTargetLowering:: combineElementTruncationToVectorTruncation(SDNode *N, DAGCombinerInfo &DCI) const { assert(N->getOpcode() == ISD::BUILD_VECTOR && "Should be called with a BUILD_VECTOR node"); SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); SDValue FirstInput = N->getOperand(0); assert(FirstInput.getOpcode() == PPCISD::MFVSR && "The input operand must be an fp-to-int conversion."); // This combine happens after legalization so the fp_to_[su]i nodes are // already converted to PPCSISD nodes. unsigned FirstConversion = FirstInput.getOperand(0).getOpcode(); if (FirstConversion == PPCISD::FCTIDZ || FirstConversion == PPCISD::FCTIDUZ || FirstConversion == PPCISD::FCTIWZ || FirstConversion == PPCISD::FCTIWUZ) { bool IsSplat = true; bool Is32Bit = FirstConversion == PPCISD::FCTIWZ || FirstConversion == PPCISD::FCTIWUZ; EVT SrcVT = FirstInput.getOperand(0).getValueType(); SmallVector Ops; EVT TargetVT = N->getValueType(0); for (int i = 0, e = N->getNumOperands(); i < e; ++i) { SDValue NextOp = N->getOperand(i); if (NextOp.getOpcode() != PPCISD::MFVSR) return SDValue(); unsigned NextConversion = NextOp.getOperand(0).getOpcode(); if (NextConversion != FirstConversion) return SDValue(); // If we are converting to 32-bit integers, we need to add an FP_ROUND. // This is not valid if the input was originally double precision. It is // also not profitable to do unless this is an extending load in which // case doing this combine will allow us to combine consecutive loads. if (Is32Bit && !isFPExtLoad(NextOp.getOperand(0).getOperand(0))) return SDValue(); if (N->getOperand(i) != FirstInput) IsSplat = false; } // If this is a splat, we leave it as-is since there will be only a single // fp-to-int conversion followed by a splat of the integer. This is better // for 32-bit and smaller ints and neutral for 64-bit ints. if (IsSplat) return SDValue(); // Now that we know we have the right type of node, get its operands for (int i = 0, e = N->getNumOperands(); i < e; ++i) { SDValue In = N->getOperand(i).getOperand(0); if (Is32Bit) { // For 32-bit values, we need to add an FP_ROUND node (if we made it // here, we know that all inputs are extending loads so this is safe). if (In.isUndef()) Ops.push_back(DAG.getUNDEF(SrcVT)); else { SDValue Trunc = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, In.getOperand(0), DAG.getIntPtrConstant(1, dl)); Ops.push_back(Trunc); } } else Ops.push_back(In.isUndef() ? DAG.getUNDEF(SrcVT) : In.getOperand(0)); } unsigned Opcode; if (FirstConversion == PPCISD::FCTIDZ || FirstConversion == PPCISD::FCTIWZ) Opcode = ISD::FP_TO_SINT; else Opcode = ISD::FP_TO_UINT; EVT NewVT = TargetVT == MVT::v2i64 ? MVT::v2f64 : MVT::v4f32; SDValue BV = DAG.getBuildVector(NewVT, dl, Ops); return DAG.getNode(Opcode, dl, TargetVT, BV); } return SDValue(); } /// Reduce the number of loads when building a vector. /// /// Building a vector out of multiple loads can be converted to a load /// of the vector type if the loads are consecutive. If the loads are /// consecutive but in descending order, a shuffle is added at the end /// to reorder the vector. static SDValue combineBVOfConsecutiveLoads(SDNode *N, SelectionDAG &DAG) { assert(N->getOpcode() == ISD::BUILD_VECTOR && "Should be called with a BUILD_VECTOR node"); SDLoc dl(N); // Return early for non byte-sized type, as they can't be consecutive. if (!N->getValueType(0).getVectorElementType().isByteSized()) return SDValue(); bool InputsAreConsecutiveLoads = true; bool InputsAreReverseConsecutive = true; unsigned ElemSize = N->getValueType(0).getScalarType().getStoreSize(); SDValue FirstInput = N->getOperand(0); bool IsRoundOfExtLoad = false; if (FirstInput.getOpcode() == ISD::FP_ROUND && FirstInput.getOperand(0).getOpcode() == ISD::LOAD) { LoadSDNode *LD = dyn_cast(FirstInput.getOperand(0)); IsRoundOfExtLoad = LD->getExtensionType() == ISD::EXTLOAD; } // Not a build vector of (possibly fp_rounded) loads. if ((!IsRoundOfExtLoad && FirstInput.getOpcode() != ISD::LOAD) || N->getNumOperands() == 1) return SDValue(); for (int i = 1, e = N->getNumOperands(); i < e; ++i) { // If any inputs are fp_round(extload), they all must be. if (IsRoundOfExtLoad && N->getOperand(i).getOpcode() != ISD::FP_ROUND) return SDValue(); SDValue NextInput = IsRoundOfExtLoad ? N->getOperand(i).getOperand(0) : N->getOperand(i); if (NextInput.getOpcode() != ISD::LOAD) return SDValue(); SDValue PreviousInput = IsRoundOfExtLoad ? N->getOperand(i-1).getOperand(0) : N->getOperand(i-1); LoadSDNode *LD1 = dyn_cast(PreviousInput); LoadSDNode *LD2 = dyn_cast(NextInput); // If any inputs are fp_round(extload), they all must be. if (IsRoundOfExtLoad && LD2->getExtensionType() != ISD::EXTLOAD) return SDValue(); if (!isConsecutiveLS(LD2, LD1, ElemSize, 1, DAG)) InputsAreConsecutiveLoads = false; if (!isConsecutiveLS(LD1, LD2, ElemSize, 1, DAG)) InputsAreReverseConsecutive = false; // Exit early if the loads are neither consecutive nor reverse consecutive. if (!InputsAreConsecutiveLoads && !InputsAreReverseConsecutive) return SDValue(); } assert(!(InputsAreConsecutiveLoads && InputsAreReverseConsecutive) && "The loads cannot be both consecutive and reverse consecutive."); SDValue FirstLoadOp = IsRoundOfExtLoad ? FirstInput.getOperand(0) : FirstInput; SDValue LastLoadOp = IsRoundOfExtLoad ? N->getOperand(N->getNumOperands()-1).getOperand(0) : N->getOperand(N->getNumOperands()-1); LoadSDNode *LD1 = dyn_cast(FirstLoadOp); LoadSDNode *LDL = dyn_cast(LastLoadOp); if (InputsAreConsecutiveLoads) { assert(LD1 && "Input needs to be a LoadSDNode."); return DAG.getLoad(N->getValueType(0), dl, LD1->getChain(), LD1->getBasePtr(), LD1->getPointerInfo(), LD1->getAlignment()); } if (InputsAreReverseConsecutive) { assert(LDL && "Input needs to be a LoadSDNode."); SDValue Load = DAG.getLoad(N->getValueType(0), dl, LDL->getChain(), LDL->getBasePtr(), LDL->getPointerInfo(), LDL->getAlignment()); SmallVector Ops; for (int i = N->getNumOperands() - 1; i >= 0; i--) Ops.push_back(i); return DAG.getVectorShuffle(N->getValueType(0), dl, Load, DAG.getUNDEF(N->getValueType(0)), Ops); } return SDValue(); } // This function adds the required vector_shuffle needed to get // the elements of the vector extract in the correct position // as specified by the CorrectElems encoding. static SDValue addShuffleForVecExtend(SDNode *N, SelectionDAG &DAG, SDValue Input, uint64_t Elems, uint64_t CorrectElems) { SDLoc dl(N); unsigned NumElems = Input.getValueType().getVectorNumElements(); SmallVector ShuffleMask(NumElems, -1); // Knowing the element indices being extracted from the original // vector and the order in which they're being inserted, just put // them at element indices required for the instruction. for (unsigned i = 0; i < N->getNumOperands(); i++) { if (DAG.getDataLayout().isLittleEndian()) ShuffleMask[CorrectElems & 0xF] = Elems & 0xF; else ShuffleMask[(CorrectElems & 0xF0) >> 4] = (Elems & 0xF0) >> 4; CorrectElems = CorrectElems >> 8; Elems = Elems >> 8; } SDValue Shuffle = DAG.getVectorShuffle(Input.getValueType(), dl, Input, DAG.getUNDEF(Input.getValueType()), ShuffleMask); EVT Ty = N->getValueType(0); SDValue BV = DAG.getNode(PPCISD::SExtVElems, dl, Ty, Shuffle); return BV; } // Look for build vector patterns where input operands come from sign // extended vector_extract elements of specific indices. If the correct indices // aren't used, add a vector shuffle to fix up the indices and create a new // PPCISD:SExtVElems node which selects the vector sign extend instructions // during instruction selection. static SDValue combineBVOfVecSExt(SDNode *N, SelectionDAG &DAG) { // This array encodes the indices that the vector sign extend instructions // extract from when extending from one type to another for both BE and LE. // The right nibble of each byte corresponds to the LE incides. // and the left nibble of each byte corresponds to the BE incides. // For example: 0x3074B8FC byte->word // For LE: the allowed indices are: 0x0,0x4,0x8,0xC // For BE: the allowed indices are: 0x3,0x7,0xB,0xF // For example: 0x000070F8 byte->double word // For LE: the allowed indices are: 0x0,0x8 // For BE: the allowed indices are: 0x7,0xF uint64_t TargetElems[] = { 0x3074B8FC, // b->w 0x000070F8, // b->d 0x10325476, // h->w 0x00003074, // h->d 0x00001032, // w->d }; uint64_t Elems = 0; int Index; SDValue Input; auto isSExtOfVecExtract = [&](SDValue Op) -> bool { if (!Op) return false; if (Op.getOpcode() != ISD::SIGN_EXTEND && Op.getOpcode() != ISD::SIGN_EXTEND_INREG) return false; // A SIGN_EXTEND_INREG might be fed by an ANY_EXTEND to produce a value // of the right width. SDValue Extract = Op.getOperand(0); if (Extract.getOpcode() == ISD::ANY_EXTEND) Extract = Extract.getOperand(0); if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT) return false; ConstantSDNode *ExtOp = dyn_cast(Extract.getOperand(1)); if (!ExtOp) return false; Index = ExtOp->getZExtValue(); if (Input && Input != Extract.getOperand(0)) return false; if (!Input) Input = Extract.getOperand(0); Elems = Elems << 8; Index = DAG.getDataLayout().isLittleEndian() ? Index : Index << 4; Elems |= Index; return true; }; // If the build vector operands aren't sign extended vector extracts, // of the same input vector, then return. for (unsigned i = 0; i < N->getNumOperands(); i++) { if (!isSExtOfVecExtract(N->getOperand(i))) { return SDValue(); } } // If the vector extract indicies are not correct, add the appropriate // vector_shuffle. int TgtElemArrayIdx; int InputSize = Input.getValueType().getScalarSizeInBits(); int OutputSize = N->getValueType(0).getScalarSizeInBits(); if (InputSize + OutputSize == 40) TgtElemArrayIdx = 0; else if (InputSize + OutputSize == 72) TgtElemArrayIdx = 1; else if (InputSize + OutputSize == 48) TgtElemArrayIdx = 2; else if (InputSize + OutputSize == 80) TgtElemArrayIdx = 3; else if (InputSize + OutputSize == 96) TgtElemArrayIdx = 4; else return SDValue(); uint64_t CorrectElems = TargetElems[TgtElemArrayIdx]; CorrectElems = DAG.getDataLayout().isLittleEndian() ? CorrectElems & 0x0F0F0F0F0F0F0F0F : CorrectElems & 0xF0F0F0F0F0F0F0F0; if (Elems != CorrectElems) { return addShuffleForVecExtend(N, DAG, Input, Elems, CorrectElems); } // Regular lowering will catch cases where a shuffle is not needed. return SDValue(); } SDValue PPCTargetLowering::DAGCombineBuildVector(SDNode *N, DAGCombinerInfo &DCI) const { assert(N->getOpcode() == ISD::BUILD_VECTOR && "Should be called with a BUILD_VECTOR node"); SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); if (!Subtarget.hasVSX()) return SDValue(); // The target independent DAG combiner will leave a build_vector of // float-to-int conversions intact. We can generate MUCH better code for // a float-to-int conversion of a vector of floats. SDValue FirstInput = N->getOperand(0); if (FirstInput.getOpcode() == PPCISD::MFVSR) { SDValue Reduced = combineElementTruncationToVectorTruncation(N, DCI); if (Reduced) return Reduced; } // If we're building a vector out of consecutive loads, just load that // vector type. SDValue Reduced = combineBVOfConsecutiveLoads(N, DAG); if (Reduced) return Reduced; // If we're building a vector out of extended elements from another vector // we have P9 vector integer extend instructions. The code assumes legal // input types (i.e. it can't handle things like v4i16) so do not run before // legalization. if (Subtarget.hasP9Altivec() && !DCI.isBeforeLegalize()) { Reduced = combineBVOfVecSExt(N, DAG); if (Reduced) return Reduced; } if (N->getValueType(0) != MVT::v2f64) return SDValue(); // Looking for: // (build_vector ([su]int_to_fp (extractelt 0)), [su]int_to_fp (extractelt 1)) if (FirstInput.getOpcode() != ISD::SINT_TO_FP && FirstInput.getOpcode() != ISD::UINT_TO_FP) return SDValue(); if (N->getOperand(1).getOpcode() != ISD::SINT_TO_FP && N->getOperand(1).getOpcode() != ISD::UINT_TO_FP) return SDValue(); if (FirstInput.getOpcode() != N->getOperand(1).getOpcode()) return SDValue(); SDValue Ext1 = FirstInput.getOperand(0); SDValue Ext2 = N->getOperand(1).getOperand(0); if(Ext1.getOpcode() != ISD::EXTRACT_VECTOR_ELT || Ext2.getOpcode() != ISD::EXTRACT_VECTOR_ELT) return SDValue(); ConstantSDNode *Ext1Op = dyn_cast(Ext1.getOperand(1)); ConstantSDNode *Ext2Op = dyn_cast(Ext2.getOperand(1)); if (!Ext1Op || !Ext2Op) return SDValue(); if (Ext1.getOperand(0).getValueType() != MVT::v4i32 || Ext1.getOperand(0) != Ext2.getOperand(0)) return SDValue(); int FirstElem = Ext1Op->getZExtValue(); int SecondElem = Ext2Op->getZExtValue(); int SubvecIdx; if (FirstElem == 0 && SecondElem == 1) SubvecIdx = Subtarget.isLittleEndian() ? 1 : 0; else if (FirstElem == 2 && SecondElem == 3) SubvecIdx = Subtarget.isLittleEndian() ? 0 : 1; else return SDValue(); SDValue SrcVec = Ext1.getOperand(0); auto NodeType = (N->getOperand(1).getOpcode() == ISD::SINT_TO_FP) ? PPCISD::SINT_VEC_TO_FP : PPCISD::UINT_VEC_TO_FP; return DAG.getNode(NodeType, dl, MVT::v2f64, SrcVec, DAG.getIntPtrConstant(SubvecIdx, dl)); } SDValue PPCTargetLowering::combineFPToIntToFP(SDNode *N, DAGCombinerInfo &DCI) const { assert((N->getOpcode() == ISD::SINT_TO_FP || N->getOpcode() == ISD::UINT_TO_FP) && "Need an int -> FP conversion node here"); if (useSoftFloat() || !Subtarget.has64BitSupport()) return SDValue(); SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); SDValue Op(N, 0); // Don't handle ppc_fp128 here or conversions that are out-of-range capable // from the hardware. if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64) return SDValue(); if (Op.getOperand(0).getValueType().getSimpleVT() <= MVT(MVT::i1) || Op.getOperand(0).getValueType().getSimpleVT() > MVT(MVT::i64)) return SDValue(); SDValue FirstOperand(Op.getOperand(0)); bool SubWordLoad = FirstOperand.getOpcode() == ISD::LOAD && (FirstOperand.getValueType() == MVT::i8 || FirstOperand.getValueType() == MVT::i16); if (Subtarget.hasP9Vector() && Subtarget.hasP9Altivec() && SubWordLoad) { bool Signed = N->getOpcode() == ISD::SINT_TO_FP; bool DstDouble = Op.getValueType() == MVT::f64; unsigned ConvOp = Signed ? (DstDouble ? PPCISD::FCFID : PPCISD::FCFIDS) : (DstDouble ? PPCISD::FCFIDU : PPCISD::FCFIDUS); SDValue WidthConst = DAG.getIntPtrConstant(FirstOperand.getValueType() == MVT::i8 ? 1 : 2, dl, false); LoadSDNode *LDN = cast(FirstOperand.getNode()); SDValue Ops[] = { LDN->getChain(), LDN->getBasePtr(), WidthConst }; SDValue Ld = DAG.getMemIntrinsicNode(PPCISD::LXSIZX, dl, DAG.getVTList(MVT::f64, MVT::Other), Ops, MVT::i8, LDN->getMemOperand()); // For signed conversion, we need to sign-extend the value in the VSR if (Signed) { SDValue ExtOps[] = { Ld, WidthConst }; SDValue Ext = DAG.getNode(PPCISD::VEXTS, dl, MVT::f64, ExtOps); return DAG.getNode(ConvOp, dl, DstDouble ? MVT::f64 : MVT::f32, Ext); } else return DAG.getNode(ConvOp, dl, DstDouble ? MVT::f64 : MVT::f32, Ld); } // For i32 intermediate values, unfortunately, the conversion functions // leave the upper 32 bits of the value are undefined. Within the set of // scalar instructions, we have no method for zero- or sign-extending the // value. Thus, we cannot handle i32 intermediate values here. if (Op.getOperand(0).getValueType() == MVT::i32) return SDValue(); assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) && "UINT_TO_FP is supported only with FPCVT"); // If we have FCFIDS, then use it when converting to single-precision. // Otherwise, convert to double-precision and then round. unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS : PPCISD::FCFIDS) : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU : PPCISD::FCFID); MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) ? MVT::f32 : MVT::f64; // If we're converting from a float, to an int, and back to a float again, // then we don't need the store/load pair at all. if ((Op.getOperand(0).getOpcode() == ISD::FP_TO_UINT && Subtarget.hasFPCVT()) || (Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT)) { SDValue Src = Op.getOperand(0).getOperand(0); if (Src.getValueType() == MVT::f32) { Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); DCI.AddToWorklist(Src.getNode()); } else if (Src.getValueType() != MVT::f64) { // Make sure that we don't pick up a ppc_fp128 source value. return SDValue(); } unsigned FCTOp = Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIDZ : PPCISD::FCTIDUZ; SDValue Tmp = DAG.getNode(FCTOp, dl, MVT::f64, Src); SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Tmp); if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) { FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, DAG.getIntPtrConstant(0, dl)); DCI.AddToWorklist(FP.getNode()); } return FP; } return SDValue(); } // expandVSXLoadForLE - Convert VSX loads (which may be intrinsics for // builtins) into loads with swaps. SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); SDValue Chain; SDValue Base; MachineMemOperand *MMO; switch (N->getOpcode()) { default: llvm_unreachable("Unexpected opcode for little endian VSX load"); case ISD::LOAD: { LoadSDNode *LD = cast(N); Chain = LD->getChain(); Base = LD->getBasePtr(); MMO = LD->getMemOperand(); // If the MMO suggests this isn't a load of a full vector, leave // things alone. For a built-in, we have to make the change for // correctness, so if there is a size problem that will be a bug. if (MMO->getSize() < 16) return SDValue(); break; } case ISD::INTRINSIC_W_CHAIN: { MemIntrinsicSDNode *Intrin = cast(N); Chain = Intrin->getChain(); // Similarly to the store case below, Intrin->getBasePtr() doesn't get // us what we want. Get operand 2 instead. Base = Intrin->getOperand(2); MMO = Intrin->getMemOperand(); break; } } MVT VecTy = N->getValueType(0).getSimpleVT(); // Do not expand to PPCISD::LXVD2X + PPCISD::XXSWAPD when the load is // aligned and the type is a vector with elements up to 4 bytes if (Subtarget.needsSwapsForVSXMemOps() && !(MMO->getAlignment()%16) && VecTy.getScalarSizeInBits() <= 32 ) { return SDValue(); } SDValue LoadOps[] = { Chain, Base }; SDValue Load = DAG.getMemIntrinsicNode(PPCISD::LXVD2X, dl, DAG.getVTList(MVT::v2f64, MVT::Other), LoadOps, MVT::v2f64, MMO); DCI.AddToWorklist(Load.getNode()); Chain = Load.getValue(1); SDValue Swap = DAG.getNode( PPCISD::XXSWAPD, dl, DAG.getVTList(MVT::v2f64, MVT::Other), Chain, Load); DCI.AddToWorklist(Swap.getNode()); // Add a bitcast if the resulting load type doesn't match v2f64. if (VecTy != MVT::v2f64) { SDValue N = DAG.getNode(ISD::BITCAST, dl, VecTy, Swap); DCI.AddToWorklist(N.getNode()); // Package {bitcast value, swap's chain} to match Load's shape. return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(VecTy, MVT::Other), N, Swap.getValue(1)); } return Swap; } // expandVSXStoreForLE - Convert VSX stores (which may be intrinsics for // builtins) into stores with swaps. SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); SDValue Chain; SDValue Base; unsigned SrcOpnd; MachineMemOperand *MMO; switch (N->getOpcode()) { default: llvm_unreachable("Unexpected opcode for little endian VSX store"); case ISD::STORE: { StoreSDNode *ST = cast(N); Chain = ST->getChain(); Base = ST->getBasePtr(); MMO = ST->getMemOperand(); SrcOpnd = 1; // If the MMO suggests this isn't a store of a full vector, leave // things alone. For a built-in, we have to make the change for // correctness, so if there is a size problem that will be a bug. if (MMO->getSize() < 16) return SDValue(); break; } case ISD::INTRINSIC_VOID: { MemIntrinsicSDNode *Intrin = cast(N); Chain = Intrin->getChain(); // Intrin->getBasePtr() oddly does not get what we want. Base = Intrin->getOperand(3); MMO = Intrin->getMemOperand(); SrcOpnd = 2; break; } } SDValue Src = N->getOperand(SrcOpnd); MVT VecTy = Src.getValueType().getSimpleVT(); // Do not expand to PPCISD::XXSWAPD and PPCISD::STXVD2X when the load is // aligned and the type is a vector with elements up to 4 bytes if (Subtarget.needsSwapsForVSXMemOps() && !(MMO->getAlignment()%16) && VecTy.getScalarSizeInBits() <= 32 ) { return SDValue(); } // All stores are done as v2f64 and possible bit cast. if (VecTy != MVT::v2f64) { Src = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Src); DCI.AddToWorklist(Src.getNode()); } SDValue Swap = DAG.getNode(PPCISD::XXSWAPD, dl, DAG.getVTList(MVT::v2f64, MVT::Other), Chain, Src); DCI.AddToWorklist(Swap.getNode()); Chain = Swap.getValue(1); SDValue StoreOps[] = { Chain, Swap, Base }; SDValue Store = DAG.getMemIntrinsicNode(PPCISD::STXVD2X, dl, DAG.getVTList(MVT::Other), StoreOps, VecTy, MMO); DCI.AddToWorklist(Store.getNode()); return Store; } // Handle DAG combine for STORE (FP_TO_INT F). SDValue PPCTargetLowering::combineStoreFPToInt(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); unsigned Opcode = N->getOperand(1).getOpcode(); assert((Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT) && "Not a FP_TO_INT Instruction!"); SDValue Val = N->getOperand(1).getOperand(0); EVT Op1VT = N->getOperand(1).getValueType(); EVT ResVT = Val.getValueType(); // Floating point types smaller than 32 bits are not legal on Power. if (ResVT.getScalarSizeInBits() < 32) return SDValue(); // Only perform combine for conversion to i64/i32 or power9 i16/i8. bool ValidTypeForStoreFltAsInt = (Op1VT == MVT::i32 || Op1VT == MVT::i64 || (Subtarget.hasP9Vector() && (Op1VT == MVT::i16 || Op1VT == MVT::i8))); if (ResVT == MVT::ppcf128 || !Subtarget.hasP8Altivec() || cast(N)->isTruncatingStore() || !ValidTypeForStoreFltAsInt) return SDValue(); // Extend f32 values to f64 if (ResVT.getScalarSizeInBits() == 32) { Val = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Val); DCI.AddToWorklist(Val.getNode()); } // Set signed or unsigned conversion opcode. unsigned ConvOpcode = (Opcode == ISD::FP_TO_SINT) ? PPCISD::FP_TO_SINT_IN_VSR : PPCISD::FP_TO_UINT_IN_VSR; Val = DAG.getNode(ConvOpcode, dl, ResVT == MVT::f128 ? MVT::f128 : MVT::f64, Val); DCI.AddToWorklist(Val.getNode()); // Set number of bytes being converted. unsigned ByteSize = Op1VT.getScalarSizeInBits() / 8; SDValue Ops[] = { N->getOperand(0), Val, N->getOperand(2), DAG.getIntPtrConstant(ByteSize, dl, false), DAG.getValueType(Op1VT) }; Val = DAG.getMemIntrinsicNode(PPCISD::ST_VSR_SCAL_INT, dl, DAG.getVTList(MVT::Other), Ops, cast(N)->getMemoryVT(), cast(N)->getMemOperand()); DCI.AddToWorklist(Val.getNode()); return Val; } SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); switch (N->getOpcode()) { default: break; case ISD::ADD: return combineADD(N, DCI); case ISD::SHL: return combineSHL(N, DCI); case ISD::SRA: return combineSRA(N, DCI); case ISD::SRL: return combineSRL(N, DCI); case ISD::MUL: return combineMUL(N, DCI); case PPCISD::SHL: if (isNullConstant(N->getOperand(0))) // 0 << V -> 0. return N->getOperand(0); break; case PPCISD::SRL: if (isNullConstant(N->getOperand(0))) // 0 >>u V -> 0. return N->getOperand(0); break; case PPCISD::SRA: if (ConstantSDNode *C = dyn_cast(N->getOperand(0))) { if (C->isNullValue() || // 0 >>s V -> 0. C->isAllOnesValue()) // -1 >>s V -> -1. return N->getOperand(0); } break; case ISD::SIGN_EXTEND: case ISD::ZERO_EXTEND: case ISD::ANY_EXTEND: return DAGCombineExtBoolTrunc(N, DCI); case ISD::TRUNCATE: return combineTRUNCATE(N, DCI); case ISD::SETCC: if (SDValue CSCC = combineSetCC(N, DCI)) return CSCC; LLVM_FALLTHROUGH; case ISD::SELECT_CC: return DAGCombineTruncBoolExt(N, DCI); case ISD::SINT_TO_FP: case ISD::UINT_TO_FP: return combineFPToIntToFP(N, DCI); case ISD::STORE: { EVT Op1VT = N->getOperand(1).getValueType(); unsigned Opcode = N->getOperand(1).getOpcode(); if (Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT) { SDValue Val= combineStoreFPToInt(N, DCI); if (Val) return Val; } // Turn STORE (BSWAP) -> sthbrx/stwbrx. if (cast(N)->isUnindexed() && Opcode == ISD::BSWAP && N->getOperand(1).getNode()->hasOneUse() && (Op1VT == MVT::i32 || Op1VT == MVT::i16 || (Subtarget.hasLDBRX() && Subtarget.isPPC64() && Op1VT == MVT::i64))) { // STBRX can only handle simple types and it makes no sense to store less // two bytes in byte-reversed order. EVT mVT = cast(N)->getMemoryVT(); if (mVT.isExtended() || mVT.getSizeInBits() < 16) break; SDValue BSwapOp = N->getOperand(1).getOperand(0); // Do an any-extend to 32-bits if this is a half-word input. if (BSwapOp.getValueType() == MVT::i16) BSwapOp = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, BSwapOp); // If the type of BSWAP operand is wider than stored memory width // it need to be shifted to the right side before STBRX. if (Op1VT.bitsGT(mVT)) { int Shift = Op1VT.getSizeInBits() - mVT.getSizeInBits(); BSwapOp = DAG.getNode(ISD::SRL, dl, Op1VT, BSwapOp, DAG.getConstant(Shift, dl, MVT::i32)); // Need to truncate if this is a bswap of i64 stored as i32/i16. if (Op1VT == MVT::i64) BSwapOp = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, BSwapOp); } SDValue Ops[] = { N->getOperand(0), BSwapOp, N->getOperand(2), DAG.getValueType(mVT) }; return DAG.getMemIntrinsicNode(PPCISD::STBRX, dl, DAG.getVTList(MVT::Other), Ops, cast(N)->getMemoryVT(), cast(N)->getMemOperand()); } // STORE Constant:i32<0> -> STORE Constant:i64<0> // So it can increase the chance of CSE constant construction. if (Subtarget.isPPC64() && !DCI.isBeforeLegalize() && isa(N->getOperand(1)) && Op1VT == MVT::i32) { // Need to sign-extended to 64-bits to handle negative values. EVT MemVT = cast(N)->getMemoryVT(); uint64_t Val64 = SignExtend64(N->getConstantOperandVal(1), MemVT.getSizeInBits()); SDValue Const64 = DAG.getConstant(Val64, dl, MVT::i64); // DAG.getTruncStore() can't be used here because it doesn't accept // the general (base + offset) addressing mode. // So we use UpdateNodeOperands and setTruncatingStore instead. DAG.UpdateNodeOperands(N, N->getOperand(0), Const64, N->getOperand(2), N->getOperand(3)); cast(N)->setTruncatingStore(true); return SDValue(N, 0); } // For little endian, VSX stores require generating xxswapd/lxvd2x. // Not needed on ISA 3.0 based CPUs since we have a non-permuting store. if (Op1VT.isSimple()) { MVT StoreVT = Op1VT.getSimpleVT(); if (Subtarget.needsSwapsForVSXMemOps() && (StoreVT == MVT::v2f64 || StoreVT == MVT::v2i64 || StoreVT == MVT::v4f32 || StoreVT == MVT::v4i32)) return expandVSXStoreForLE(N, DCI); } break; } case ISD::LOAD: { LoadSDNode *LD = cast(N); EVT VT = LD->getValueType(0); // For little endian, VSX loads require generating lxvd2x/xxswapd. // Not needed on ISA 3.0 based CPUs since we have a non-permuting load. if (VT.isSimple()) { MVT LoadVT = VT.getSimpleVT(); if (Subtarget.needsSwapsForVSXMemOps() && (LoadVT == MVT::v2f64 || LoadVT == MVT::v2i64 || LoadVT == MVT::v4f32 || LoadVT == MVT::v4i32)) return expandVSXLoadForLE(N, DCI); } // We sometimes end up with a 64-bit integer load, from which we extract // two single-precision floating-point numbers. This happens with // std::complex, and other similar structures, because of the way we // canonicalize structure copies. However, if we lack direct moves, // then the final bitcasts from the extracted integer values to the // floating-point numbers turn into store/load pairs. Even with direct moves, // just loading the two floating-point numbers is likely better. auto ReplaceTwoFloatLoad = [&]() { if (VT != MVT::i64) return false; if (LD->getExtensionType() != ISD::NON_EXTLOAD || LD->isVolatile()) return false; // We're looking for a sequence like this: // t13: i64,ch = load t0, t6, undef:i64 // t16: i64 = srl t13, Constant:i32<32> // t17: i32 = truncate t16 // t18: f32 = bitcast t17 // t19: i32 = truncate t13 // t20: f32 = bitcast t19 if (!LD->hasNUsesOfValue(2, 0)) return false; auto UI = LD->use_begin(); while (UI.getUse().getResNo() != 0) ++UI; SDNode *Trunc = *UI++; while (UI.getUse().getResNo() != 0) ++UI; SDNode *RightShift = *UI; if (Trunc->getOpcode() != ISD::TRUNCATE) std::swap(Trunc, RightShift); if (Trunc->getOpcode() != ISD::TRUNCATE || Trunc->getValueType(0) != MVT::i32 || !Trunc->hasOneUse()) return false; if (RightShift->getOpcode() != ISD::SRL || !isa(RightShift->getOperand(1)) || RightShift->getConstantOperandVal(1) != 32 || !RightShift->hasOneUse()) return false; SDNode *Trunc2 = *RightShift->use_begin(); if (Trunc2->getOpcode() != ISD::TRUNCATE || Trunc2->getValueType(0) != MVT::i32 || !Trunc2->hasOneUse()) return false; SDNode *Bitcast = *Trunc->use_begin(); SDNode *Bitcast2 = *Trunc2->use_begin(); if (Bitcast->getOpcode() != ISD::BITCAST || Bitcast->getValueType(0) != MVT::f32) return false; if (Bitcast2->getOpcode() != ISD::BITCAST || Bitcast2->getValueType(0) != MVT::f32) return false; if (Subtarget.isLittleEndian()) std::swap(Bitcast, Bitcast2); // Bitcast has the second float (in memory-layout order) and Bitcast2 // has the first one. SDValue BasePtr = LD->getBasePtr(); if (LD->isIndexed()) { assert(LD->getAddressingMode() == ISD::PRE_INC && "Non-pre-inc AM on PPC?"); BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, LD->getOffset()); } auto MMOFlags = LD->getMemOperand()->getFlags() & ~MachineMemOperand::MOVolatile; SDValue FloatLoad = DAG.getLoad(MVT::f32, dl, LD->getChain(), BasePtr, LD->getPointerInfo(), LD->getAlignment(), MMOFlags, LD->getAAInfo()); SDValue AddPtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, DAG.getIntPtrConstant(4, dl)); SDValue FloatLoad2 = DAG.getLoad( MVT::f32, dl, SDValue(FloatLoad.getNode(), 1), AddPtr, LD->getPointerInfo().getWithOffset(4), MinAlign(LD->getAlignment(), 4), MMOFlags, LD->getAAInfo()); if (LD->isIndexed()) { // Note that DAGCombine should re-form any pre-increment load(s) from // what is produced here if that makes sense. DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), BasePtr); } DCI.CombineTo(Bitcast2, FloatLoad); DCI.CombineTo(Bitcast, FloatLoad2); DAG.ReplaceAllUsesOfValueWith(SDValue(LD, LD->isIndexed() ? 2 : 1), SDValue(FloatLoad2.getNode(), 1)); return true; }; if (ReplaceTwoFloatLoad()) return SDValue(N, 0); EVT MemVT = LD->getMemoryVT(); Type *Ty = MemVT.getTypeForEVT(*DAG.getContext()); unsigned ABIAlignment = DAG.getDataLayout().getABITypeAlignment(Ty); Type *STy = MemVT.getScalarType().getTypeForEVT(*DAG.getContext()); unsigned ScalarABIAlignment = DAG.getDataLayout().getABITypeAlignment(STy); if (LD->isUnindexed() && VT.isVector() && ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) && // P8 and later hardware should just use LOAD. !Subtarget.hasP8Vector() && (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v4f32)) || (Subtarget.hasQPX() && (VT == MVT::v4f64 || VT == MVT::v4f32) && LD->getAlignment() >= ScalarABIAlignment)) && LD->getAlignment() < ABIAlignment) { // This is a type-legal unaligned Altivec or QPX load. SDValue Chain = LD->getChain(); SDValue Ptr = LD->getBasePtr(); bool isLittleEndian = Subtarget.isLittleEndian(); // This implements the loading of unaligned vectors as described in // the venerable Apple Velocity Engine overview. Specifically: // https://developer.apple.com/hardwaredrivers/ve/alignment.html // https://developer.apple.com/hardwaredrivers/ve/code_optimization.html // // The general idea is to expand a sequence of one or more unaligned // loads into an alignment-based permutation-control instruction (lvsl // or lvsr), a series of regular vector loads (which always truncate // their input address to an aligned address), and a series of // permutations. The results of these permutations are the requested // loaded values. The trick is that the last "extra" load is not taken // from the address you might suspect (sizeof(vector) bytes after the // last requested load), but rather sizeof(vector) - 1 bytes after the // last requested vector. The point of this is to avoid a page fault if // the base address happened to be aligned. This works because if the // base address is aligned, then adding less than a full vector length // will cause the last vector in the sequence to be (re)loaded. // Otherwise, the next vector will be fetched as you might suspect was // necessary. // We might be able to reuse the permutation generation from // a different base address offset from this one by an aligned amount. // The INTRINSIC_WO_CHAIN DAG combine will attempt to perform this // optimization later. Intrinsic::ID Intr, IntrLD, IntrPerm; MVT PermCntlTy, PermTy, LDTy; if (Subtarget.hasAltivec()) { Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr : Intrinsic::ppc_altivec_lvsl; IntrLD = Intrinsic::ppc_altivec_lvx; IntrPerm = Intrinsic::ppc_altivec_vperm; PermCntlTy = MVT::v16i8; PermTy = MVT::v4i32; LDTy = MVT::v4i32; } else { Intr = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlpcld : Intrinsic::ppc_qpx_qvlpcls; IntrLD = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlfd : Intrinsic::ppc_qpx_qvlfs; IntrPerm = Intrinsic::ppc_qpx_qvfperm; PermCntlTy = MVT::v4f64; PermTy = MVT::v4f64; LDTy = MemVT.getSimpleVT(); } SDValue PermCntl = BuildIntrinsicOp(Intr, Ptr, DAG, dl, PermCntlTy); // Create the new MMO for the new base load. It is like the original MMO, // but represents an area in memory almost twice the vector size centered // on the original address. If the address is unaligned, we might start // reading up to (sizeof(vector)-1) bytes below the address of the // original unaligned load. MachineFunction &MF = DAG.getMachineFunction(); MachineMemOperand *BaseMMO = MF.getMachineMemOperand(LD->getMemOperand(), -(long)MemVT.getStoreSize()+1, 2*MemVT.getStoreSize()-1); // Create the new base load. SDValue LDXIntID = DAG.getTargetConstant(IntrLD, dl, getPointerTy(MF.getDataLayout())); SDValue BaseLoadOps[] = { Chain, LDXIntID, Ptr }; SDValue BaseLoad = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, DAG.getVTList(PermTy, MVT::Other), BaseLoadOps, LDTy, BaseMMO); // Note that the value of IncOffset (which is provided to the next // load's pointer info offset value, and thus used to calculate the // alignment), and the value of IncValue (which is actually used to // increment the pointer value) are different! This is because we // require the next load to appear to be aligned, even though it // is actually offset from the base pointer by a lesser amount. int IncOffset = VT.getSizeInBits() / 8; int IncValue = IncOffset; // Walk (both up and down) the chain looking for another load at the real // (aligned) offset (the alignment of the other load does not matter in // this case). If found, then do not use the offset reduction trick, as // that will prevent the loads from being later combined (as they would // otherwise be duplicates). if (!findConsecutiveLoad(LD, DAG)) --IncValue; SDValue Increment = DAG.getConstant(IncValue, dl, getPointerTy(MF.getDataLayout())); Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); MachineMemOperand *ExtraMMO = MF.getMachineMemOperand(LD->getMemOperand(), 1, 2*MemVT.getStoreSize()-1); SDValue ExtraLoadOps[] = { Chain, LDXIntID, Ptr }; SDValue ExtraLoad = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, DAG.getVTList(PermTy, MVT::Other), ExtraLoadOps, LDTy, ExtraMMO); SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, BaseLoad.getValue(1), ExtraLoad.getValue(1)); // Because vperm has a big-endian bias, we must reverse the order // of the input vectors and complement the permute control vector // when generating little endian code. We have already handled the // latter by using lvsr instead of lvsl, so just reverse BaseLoad // and ExtraLoad here. SDValue Perm; if (isLittleEndian) Perm = BuildIntrinsicOp(IntrPerm, ExtraLoad, BaseLoad, PermCntl, DAG, dl); else Perm = BuildIntrinsicOp(IntrPerm, BaseLoad, ExtraLoad, PermCntl, DAG, dl); if (VT != PermTy) Perm = Subtarget.hasAltivec() ? DAG.getNode(ISD::BITCAST, dl, VT, Perm) : DAG.getNode(ISD::FP_ROUND, dl, VT, Perm, // QPX DAG.getTargetConstant(1, dl, MVT::i64)); // second argument is 1 because this rounding // is always exact. // The output of the permutation is our loaded result, the TokenFactor is // our new chain. DCI.CombineTo(N, Perm, TF); return SDValue(N, 0); } } break; case ISD::INTRINSIC_WO_CHAIN: { bool isLittleEndian = Subtarget.isLittleEndian(); unsigned IID = cast(N->getOperand(0))->getZExtValue(); Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr : Intrinsic::ppc_altivec_lvsl); if ((IID == Intr || IID == Intrinsic::ppc_qpx_qvlpcld || IID == Intrinsic::ppc_qpx_qvlpcls) && N->getOperand(1)->getOpcode() == ISD::ADD) { SDValue Add = N->getOperand(1); int Bits = IID == Intrinsic::ppc_qpx_qvlpcld ? 5 /* 32 byte alignment */ : 4 /* 16 byte alignment */; if (DAG.MaskedValueIsZero(Add->getOperand(1), APInt::getAllOnesValue(Bits /* alignment */) .zext(Add.getScalarValueSizeInBits()))) { SDNode *BasePtr = Add->getOperand(0).getNode(); for (SDNode::use_iterator UI = BasePtr->use_begin(), UE = BasePtr->use_end(); UI != UE; ++UI) { if (UI->getOpcode() == ISD::INTRINSIC_WO_CHAIN && cast(UI->getOperand(0))->getZExtValue() == IID) { // We've found another LVSL/LVSR, and this address is an aligned // multiple of that one. The results will be the same, so use the // one we've just found instead. return SDValue(*UI, 0); } } } if (isa(Add->getOperand(1))) { SDNode *BasePtr = Add->getOperand(0).getNode(); for (SDNode::use_iterator UI = BasePtr->use_begin(), UE = BasePtr->use_end(); UI != UE; ++UI) { if (UI->getOpcode() == ISD::ADD && isa(UI->getOperand(1)) && (cast(Add->getOperand(1))->getZExtValue() - cast(UI->getOperand(1))->getZExtValue()) % (1ULL << Bits) == 0) { SDNode *OtherAdd = *UI; for (SDNode::use_iterator VI = OtherAdd->use_begin(), VE = OtherAdd->use_end(); VI != VE; ++VI) { if (VI->getOpcode() == ISD::INTRINSIC_WO_CHAIN && cast(VI->getOperand(0))->getZExtValue() == IID) { return SDValue(*VI, 0); } } } } } } // Combine vmaxsw/h/b(a, a's negation) to abs(a) // Expose the vabsduw/h/b opportunity for down stream if (!DCI.isAfterLegalizeDAG() && Subtarget.hasP9Altivec() && (IID == Intrinsic::ppc_altivec_vmaxsw || IID == Intrinsic::ppc_altivec_vmaxsh || IID == Intrinsic::ppc_altivec_vmaxsb)) { SDValue V1 = N->getOperand(1); SDValue V2 = N->getOperand(2); if ((V1.getSimpleValueType() == MVT::v4i32 || V1.getSimpleValueType() == MVT::v8i16 || V1.getSimpleValueType() == MVT::v16i8) && V1.getSimpleValueType() == V2.getSimpleValueType()) { // (0-a, a) if (V1.getOpcode() == ISD::SUB && ISD::isBuildVectorAllZeros(V1.getOperand(0).getNode()) && V1.getOperand(1) == V2) { return DAG.getNode(ISD::ABS, dl, V2.getValueType(), V2); } // (a, 0-a) if (V2.getOpcode() == ISD::SUB && ISD::isBuildVectorAllZeros(V2.getOperand(0).getNode()) && V2.getOperand(1) == V1) { return DAG.getNode(ISD::ABS, dl, V1.getValueType(), V1); } // (x-y, y-x) if (V1.getOpcode() == ISD::SUB && V2.getOpcode() == ISD::SUB && V1.getOperand(0) == V2.getOperand(1) && V1.getOperand(1) == V2.getOperand(0)) { return DAG.getNode(ISD::ABS, dl, V1.getValueType(), V1); } } } } break; case ISD::INTRINSIC_W_CHAIN: // For little endian, VSX loads require generating lxvd2x/xxswapd. // Not needed on ISA 3.0 based CPUs since we have a non-permuting load. if (Subtarget.needsSwapsForVSXMemOps()) { switch (cast(N->getOperand(1))->getZExtValue()) { default: break; case Intrinsic::ppc_vsx_lxvw4x: case Intrinsic::ppc_vsx_lxvd2x: return expandVSXLoadForLE(N, DCI); } } break; case ISD::INTRINSIC_VOID: // For little endian, VSX stores require generating xxswapd/stxvd2x. // Not needed on ISA 3.0 based CPUs since we have a non-permuting store. if (Subtarget.needsSwapsForVSXMemOps()) { switch (cast(N->getOperand(1))->getZExtValue()) { default: break; case Intrinsic::ppc_vsx_stxvw4x: case Intrinsic::ppc_vsx_stxvd2x: return expandVSXStoreForLE(N, DCI); } } break; case ISD::BSWAP: // Turn BSWAP (LOAD) -> lhbrx/lwbrx. if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) && N->getOperand(0).hasOneUse() && (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i16 || (Subtarget.hasLDBRX() && Subtarget.isPPC64() && N->getValueType(0) == MVT::i64))) { SDValue Load = N->getOperand(0); LoadSDNode *LD = cast(Load); // Create the byte-swapping load. SDValue Ops[] = { LD->getChain(), // Chain LD->getBasePtr(), // Ptr DAG.getValueType(N->getValueType(0)) // VT }; SDValue BSLoad = DAG.getMemIntrinsicNode(PPCISD::LBRX, dl, DAG.getVTList(N->getValueType(0) == MVT::i64 ? MVT::i64 : MVT::i32, MVT::Other), Ops, LD->getMemoryVT(), LD->getMemOperand()); // If this is an i16 load, insert the truncate. SDValue ResVal = BSLoad; if (N->getValueType(0) == MVT::i16) ResVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, BSLoad); // First, combine the bswap away. This makes the value produced by the // load dead. DCI.CombineTo(N, ResVal); // Next, combine the load away, we give it a bogus result value but a real // chain result. The result value is dead because the bswap is dead. DCI.CombineTo(Load.getNode(), ResVal, BSLoad.getValue(1)); // Return N so it doesn't get rechecked! return SDValue(N, 0); } break; case PPCISD::VCMP: // If a VCMPo node already exists with exactly the same operands as this // node, use its result instead of this node (VCMPo computes both a CR6 and // a normal output). // if (!N->getOperand(0).hasOneUse() && !N->getOperand(1).hasOneUse() && !N->getOperand(2).hasOneUse()) { // Scan all of the users of the LHS, looking for VCMPo's that match. SDNode *VCMPoNode = nullptr; SDNode *LHSN = N->getOperand(0).getNode(); for (SDNode::use_iterator UI = LHSN->use_begin(), E = LHSN->use_end(); UI != E; ++UI) if (UI->getOpcode() == PPCISD::VCMPo && UI->getOperand(1) == N->getOperand(1) && UI->getOperand(2) == N->getOperand(2) && UI->getOperand(0) == N->getOperand(0)) { VCMPoNode = *UI; break; } // If there is no VCMPo node, or if the flag value has a single use, don't // transform this. if (!VCMPoNode || VCMPoNode->hasNUsesOfValue(0, 1)) break; // Look at the (necessarily single) use of the flag value. If it has a // chain, this transformation is more complex. Note that multiple things // could use the value result, which we should ignore. SDNode *FlagUser = nullptr; for (SDNode::use_iterator UI = VCMPoNode->use_begin(); FlagUser == nullptr; ++UI) { assert(UI != VCMPoNode->use_end() && "Didn't find user!"); SDNode *User = *UI; for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) { if (User->getOperand(i) == SDValue(VCMPoNode, 1)) { FlagUser = User; break; } } } // If the user is a MFOCRF instruction, we know this is safe. // Otherwise we give up for right now. if (FlagUser->getOpcode() == PPCISD::MFOCRF) return SDValue(VCMPoNode, 0); } break; case ISD::BRCOND: { SDValue Cond = N->getOperand(1); SDValue Target = N->getOperand(2); if (Cond.getOpcode() == ISD::INTRINSIC_W_CHAIN && cast(Cond.getOperand(1))->getZExtValue() == Intrinsic::loop_decrement) { // We now need to make the intrinsic dead (it cannot be instruction // selected). DAG.ReplaceAllUsesOfValueWith(Cond.getValue(1), Cond.getOperand(0)); assert(Cond.getNode()->hasOneUse() && "Counter decrement has more than one use"); return DAG.getNode(PPCISD::BDNZ, dl, MVT::Other, N->getOperand(0), Target); } } break; case ISD::BR_CC: { // If this is a branch on an altivec predicate comparison, lower this so // that we don't have to do a MFOCRF: instead, branch directly on CR6. This // lowering is done pre-legalize, because the legalizer lowers the predicate // compare down to code that is difficult to reassemble. ISD::CondCode CC = cast(N->getOperand(1))->get(); SDValue LHS = N->getOperand(2), RHS = N->getOperand(3); // Sometimes the promoted value of the intrinsic is ANDed by some non-zero // value. If so, pass-through the AND to get to the intrinsic. if (LHS.getOpcode() == ISD::AND && LHS.getOperand(0).getOpcode() == ISD::INTRINSIC_W_CHAIN && cast(LHS.getOperand(0).getOperand(1))->getZExtValue() == Intrinsic::loop_decrement && isa(LHS.getOperand(1)) && !isNullConstant(LHS.getOperand(1))) LHS = LHS.getOperand(0); if (LHS.getOpcode() == ISD::INTRINSIC_W_CHAIN && cast(LHS.getOperand(1))->getZExtValue() == Intrinsic::loop_decrement && isa(RHS)) { assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Counter decrement comparison is not EQ or NE"); unsigned Val = cast(RHS)->getZExtValue(); bool isBDNZ = (CC == ISD::SETEQ && Val) || (CC == ISD::SETNE && !Val); // We now need to make the intrinsic dead (it cannot be instruction // selected). DAG.ReplaceAllUsesOfValueWith(LHS.getValue(1), LHS.getOperand(0)); assert(LHS.getNode()->hasOneUse() && "Counter decrement has more than one use"); return DAG.getNode(isBDNZ ? PPCISD::BDNZ : PPCISD::BDZ, dl, MVT::Other, N->getOperand(0), N->getOperand(4)); } int CompareOpc; bool isDot; if (LHS.getOpcode() == ISD::INTRINSIC_WO_CHAIN && isa(RHS) && (CC == ISD::SETEQ || CC == ISD::SETNE) && getVectorCompareInfo(LHS, CompareOpc, isDot, Subtarget)) { assert(isDot && "Can't compare against a vector result!"); // If this is a comparison against something other than 0/1, then we know // that the condition is never/always true. unsigned Val = cast(RHS)->getZExtValue(); if (Val != 0 && Val != 1) { if (CC == ISD::SETEQ) // Cond never true, remove branch. return N->getOperand(0); // Always !=, turn it into an unconditional branch. return DAG.getNode(ISD::BR, dl, MVT::Other, N->getOperand(0), N->getOperand(4)); } bool BranchOnWhenPredTrue = (CC == ISD::SETEQ) ^ (Val == 0); // Create the PPCISD altivec 'dot' comparison node. SDValue Ops[] = { LHS.getOperand(2), // LHS of compare LHS.getOperand(3), // RHS of compare DAG.getConstant(CompareOpc, dl, MVT::i32) }; EVT VTs[] = { LHS.getOperand(2).getValueType(), MVT::Glue }; SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops); // Unpack the result based on how the target uses it. PPC::Predicate CompOpc; switch (cast(LHS.getOperand(1))->getZExtValue()) { default: // Can't happen, don't crash on invalid number though. case 0: // Branch on the value of the EQ bit of CR6. CompOpc = BranchOnWhenPredTrue ? PPC::PRED_EQ : PPC::PRED_NE; break; case 1: // Branch on the inverted value of the EQ bit of CR6. CompOpc = BranchOnWhenPredTrue ? PPC::PRED_NE : PPC::PRED_EQ; break; case 2: // Branch on the value of the LT bit of CR6. CompOpc = BranchOnWhenPredTrue ? PPC::PRED_LT : PPC::PRED_GE; break; case 3: // Branch on the inverted value of the LT bit of CR6. CompOpc = BranchOnWhenPredTrue ? PPC::PRED_GE : PPC::PRED_LT; break; } return DAG.getNode(PPCISD::COND_BRANCH, dl, MVT::Other, N->getOperand(0), DAG.getConstant(CompOpc, dl, MVT::i32), DAG.getRegister(PPC::CR6, MVT::i32), N->getOperand(4), CompNode.getValue(1)); } break; } case ISD::BUILD_VECTOR: return DAGCombineBuildVector(N, DCI); case ISD::ABS: return combineABS(N, DCI); case ISD::VSELECT: return combineVSelect(N, DCI); } return SDValue(); } SDValue PPCTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl &Created) const { // fold (sdiv X, pow2) EVT VT = N->getValueType(0); if (VT == MVT::i64 && !Subtarget.isPPC64()) return SDValue(); if ((VT != MVT::i32 && VT != MVT::i64) || !(Divisor.isPowerOf2() || (-Divisor).isPowerOf2())) return SDValue(); SDLoc DL(N); SDValue N0 = N->getOperand(0); bool IsNegPow2 = (-Divisor).isPowerOf2(); unsigned Lg2 = (IsNegPow2 ? -Divisor : Divisor).countTrailingZeros(); SDValue ShiftAmt = DAG.getConstant(Lg2, DL, VT); SDValue Op = DAG.getNode(PPCISD::SRA_ADDZE, DL, VT, N0, ShiftAmt); Created.push_back(Op.getNode()); if (IsNegPow2) { Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op); Created.push_back(Op.getNode()); } return Op; } //===----------------------------------------------------------------------===// // Inline Assembly Support //===----------------------------------------------------------------------===// void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const { Known.resetAll(); switch (Op.getOpcode()) { default: break; case PPCISD::LBRX: { // lhbrx is known to have the top bits cleared out. if (cast(Op.getOperand(2))->getVT() == MVT::i16) Known.Zero = 0xFFFF0000; break; } case ISD::INTRINSIC_WO_CHAIN: { switch (cast(Op.getOperand(0))->getZExtValue()) { default: break; case Intrinsic::ppc_altivec_vcmpbfp_p: case Intrinsic::ppc_altivec_vcmpeqfp_p: case Intrinsic::ppc_altivec_vcmpequb_p: case Intrinsic::ppc_altivec_vcmpequh_p: case Intrinsic::ppc_altivec_vcmpequw_p: case Intrinsic::ppc_altivec_vcmpequd_p: case Intrinsic::ppc_altivec_vcmpgefp_p: case Intrinsic::ppc_altivec_vcmpgtfp_p: case Intrinsic::ppc_altivec_vcmpgtsb_p: case Intrinsic::ppc_altivec_vcmpgtsh_p: case Intrinsic::ppc_altivec_vcmpgtsw_p: case Intrinsic::ppc_altivec_vcmpgtsd_p: case Intrinsic::ppc_altivec_vcmpgtub_p: case Intrinsic::ppc_altivec_vcmpgtuh_p: case Intrinsic::ppc_altivec_vcmpgtuw_p: case Intrinsic::ppc_altivec_vcmpgtud_p: Known.Zero = ~1U; // All bits but the low one are known to be zero. break; } } } } unsigned PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const { switch (Subtarget.getDarwinDirective()) { default: break; case PPC::DIR_970: case PPC::DIR_PWR4: case PPC::DIR_PWR5: case PPC::DIR_PWR5X: case PPC::DIR_PWR6: case PPC::DIR_PWR6X: case PPC::DIR_PWR7: case PPC::DIR_PWR8: case PPC::DIR_PWR9: { if (!ML) break; if (!DisableInnermostLoopAlign32) { // If the nested loop is an innermost loop, prefer to a 32-byte alignment, // so that we can decrease cache misses and branch-prediction misses. // Actual alignment of the loop will depend on the hotness check and other // logic in alignBlocks. if (ML->getLoopDepth() > 1 && ML->getSubLoops().empty()) return 5; } const PPCInstrInfo *TII = Subtarget.getInstrInfo(); // For small loops (between 5 and 8 instructions), align to a 32-byte // boundary so that the entire loop fits in one instruction-cache line. uint64_t LoopSize = 0; for (auto I = ML->block_begin(), IE = ML->block_end(); I != IE; ++I) for (auto J = (*I)->begin(), JE = (*I)->end(); J != JE; ++J) { LoopSize += TII->getInstSizeInBytes(*J); if (LoopSize > 32) break; } if (LoopSize > 16 && LoopSize <= 32) return 5; break; } } return TargetLowering::getPrefLoopAlignment(ML); } /// getConstraintType - Given a constraint, return the type of /// constraint it is for this target. PPCTargetLowering::ConstraintType PPCTargetLowering::getConstraintType(StringRef Constraint) const { if (Constraint.size() == 1) { switch (Constraint[0]) { default: break; case 'b': case 'r': case 'f': case 'd': case 'v': case 'y': return C_RegisterClass; case 'Z': // FIXME: While Z does indicate a memory constraint, it specifically // indicates an r+r address (used in conjunction with the 'y' modifier // in the replacement string). Currently, we're forcing the base // register to be r0 in the asm printer (which is interpreted as zero) // and forming the complete address in the second register. This is // suboptimal. return C_Memory; } } else if (Constraint == "wc") { // individual CR bits. return C_RegisterClass; } else if (Constraint == "wa" || Constraint == "wd" || Constraint == "wf" || Constraint == "ws" || Constraint == "wi" || Constraint == "ww") { return C_RegisterClass; // VSX registers. } return TargetLowering::getConstraintType(Constraint); } /// Examine constraint type and operand type and determine a weight value. /// This object must already have been set up with the operand type /// and the current alternative constraint selected. TargetLowering::ConstraintWeight PPCTargetLowering::getSingleConstraintMatchWeight( AsmOperandInfo &info, const char *constraint) const { ConstraintWeight weight = CW_Invalid; Value *CallOperandVal = info.CallOperandVal; // If we don't have a value, we can't do a match, // but allow it at the lowest weight. if (!CallOperandVal) return CW_Default; Type *type = CallOperandVal->getType(); // Look at the constraint type. if (StringRef(constraint) == "wc" && type->isIntegerTy(1)) return CW_Register; // an individual CR bit. else if ((StringRef(constraint) == "wa" || StringRef(constraint) == "wd" || StringRef(constraint) == "wf") && type->isVectorTy()) return CW_Register; else if (StringRef(constraint) == "wi" && type->isIntegerTy(64)) return CW_Register; // just hold 64-bit integers data. else if (StringRef(constraint) == "ws" && type->isDoubleTy()) return CW_Register; else if (StringRef(constraint) == "ww" && type->isFloatTy()) return CW_Register; switch (*constraint) { default: weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); break; case 'b': if (type->isIntegerTy()) weight = CW_Register; break; case 'f': if (type->isFloatTy()) weight = CW_Register; break; case 'd': if (type->isDoubleTy()) weight = CW_Register; break; case 'v': if (type->isVectorTy()) weight = CW_Register; break; case 'y': weight = CW_Register; break; case 'Z': weight = CW_Memory; break; } return weight; } std::pair PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { if (Constraint.size() == 1) { // GCC RS6000 Constraint Letters switch (Constraint[0]) { case 'b': // R1-R31 if (VT == MVT::i64 && Subtarget.isPPC64()) return std::make_pair(0U, &PPC::G8RC_NOX0RegClass); return std::make_pair(0U, &PPC::GPRC_NOR0RegClass); case 'r': // R0-R31 if (VT == MVT::i64 && Subtarget.isPPC64()) return std::make_pair(0U, &PPC::G8RCRegClass); return std::make_pair(0U, &PPC::GPRCRegClass); // 'd' and 'f' constraints are both defined to be "the floating point // registers", where one is for 32-bit and the other for 64-bit. We don't // really care overly much here so just give them all the same reg classes. case 'd': case 'f': if (Subtarget.hasSPE()) { if (VT == MVT::f32 || VT == MVT::i32) return std::make_pair(0U, &PPC::SPE4RCRegClass); if (VT == MVT::f64 || VT == MVT::i64) return std::make_pair(0U, &PPC::SPERCRegClass); } else { if (VT == MVT::f32 || VT == MVT::i32) return std::make_pair(0U, &PPC::F4RCRegClass); if (VT == MVT::f64 || VT == MVT::i64) return std::make_pair(0U, &PPC::F8RCRegClass); if (VT == MVT::v4f64 && Subtarget.hasQPX()) return std::make_pair(0U, &PPC::QFRCRegClass); if (VT == MVT::v4f32 && Subtarget.hasQPX()) return std::make_pair(0U, &PPC::QSRCRegClass); } break; case 'v': if (VT == MVT::v4f64 && Subtarget.hasQPX()) return std::make_pair(0U, &PPC::QFRCRegClass); if (VT == MVT::v4f32 && Subtarget.hasQPX()) return std::make_pair(0U, &PPC::QSRCRegClass); if (Subtarget.hasAltivec()) return std::make_pair(0U, &PPC::VRRCRegClass); break; case 'y': // crrc return std::make_pair(0U, &PPC::CRRCRegClass); } } else if (Constraint == "wc" && Subtarget.useCRBits()) { // An individual CR bit. return std::make_pair(0U, &PPC::CRBITRCRegClass); } else if ((Constraint == "wa" || Constraint == "wd" || Constraint == "wf" || Constraint == "wi") && Subtarget.hasVSX()) { return std::make_pair(0U, &PPC::VSRCRegClass); } else if ((Constraint == "ws" || Constraint == "ww") && Subtarget.hasVSX()) { if (VT == MVT::f32 && Subtarget.hasP8Vector()) return std::make_pair(0U, &PPC::VSSRCRegClass); else return std::make_pair(0U, &PPC::VSFRCRegClass); } std::pair R = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); // r[0-9]+ are used, on PPC64, to refer to the corresponding 64-bit registers // (which we call X[0-9]+). If a 64-bit value has been requested, and a // 32-bit GPR has been selected, then 'upgrade' it to the 64-bit parent // register. // FIXME: If TargetLowering::getRegForInlineAsmConstraint could somehow use // the AsmName field from *RegisterInfo.td, then this would not be necessary. if (R.first && VT == MVT::i64 && Subtarget.isPPC64() && PPC::GPRCRegClass.contains(R.first)) return std::make_pair(TRI->getMatchingSuperReg(R.first, PPC::sub_32, &PPC::G8RCRegClass), &PPC::G8RCRegClass); // GCC accepts 'cc' as an alias for 'cr0', and we need to do the same. if (!R.second && StringRef("{cc}").equals_lower(Constraint)) { R.first = PPC::CR0; R.second = &PPC::CRRCRegClass; } return R; } /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops /// vector. If it is invalid, don't add anything to Ops. void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector&Ops, SelectionDAG &DAG) const { SDValue Result; // Only support length 1 constraints. if (Constraint.length() > 1) return; char Letter = Constraint[0]; switch (Letter) { default: break; case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': { ConstantSDNode *CST = dyn_cast(Op); if (!CST) return; // Must be an immediate to match. SDLoc dl(Op); int64_t Value = CST->getSExtValue(); EVT TCVT = MVT::i64; // All constants taken to be 64 bits so that negative // numbers are printed as such. switch (Letter) { default: llvm_unreachable("Unknown constraint letter!"); case 'I': // "I" is a signed 16-bit constant. if (isInt<16>(Value)) Result = DAG.getTargetConstant(Value, dl, TCVT); break; case 'J': // "J" is a constant with only the high-order 16 bits nonzero. if (isShiftedUInt<16, 16>(Value)) Result = DAG.getTargetConstant(Value, dl, TCVT); break; case 'L': // "L" is a signed 16-bit constant shifted left 16 bits. if (isShiftedInt<16, 16>(Value)) Result = DAG.getTargetConstant(Value, dl, TCVT); break; case 'K': // "K" is a constant with only the low-order 16 bits nonzero. if (isUInt<16>(Value)) Result = DAG.getTargetConstant(Value, dl, TCVT); break; case 'M': // "M" is a constant that is greater than 31. if (Value > 31) Result = DAG.getTargetConstant(Value, dl, TCVT); break; case 'N': // "N" is a positive constant that is an exact power of two. if (Value > 0 && isPowerOf2_64(Value)) Result = DAG.getTargetConstant(Value, dl, TCVT); break; case 'O': // "O" is the constant zero. if (Value == 0) Result = DAG.getTargetConstant(Value, dl, TCVT); break; case 'P': // "P" is a constant whose negation is a signed 16-bit constant. if (isInt<16>(-Value)) Result = DAG.getTargetConstant(Value, dl, TCVT); break; } break; } } if (Result.getNode()) { Ops.push_back(Result); return; } // Handle standard constraint letters. TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); } // isLegalAddressingMode - Return true if the addressing mode represented // by AM is legal for this target, for a load/store of the specified type. bool PPCTargetLowering::isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I) const { // PPC does not allow r+i addressing modes for vectors! if (Ty->isVectorTy() && AM.BaseOffs != 0) return false; // PPC allows a sign-extended 16-bit immediate field. if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1) return false; // No global is ever allowed as a base. if (AM.BaseGV) return false; // PPC only support r+r, switch (AM.Scale) { case 0: // "r+i" or just "i", depending on HasBaseReg. break; case 1: if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed. return false; // Otherwise we have r+r or r+i. break; case 2: if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed. return false; // Allow 2*r as r+r. break; default: // No other scales are supported. return false; } return true; } SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const { MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); MFI.setReturnAddressIsTaken(true); if (verifyReturnAddressArgumentIsConstant(Op, DAG)) return SDValue(); SDLoc dl(Op); unsigned Depth = cast(Op.getOperand(0))->getZExtValue(); // Make sure the function does not optimize away the store of the RA to // the stack. PPCFunctionInfo *FuncInfo = MF.getInfo(); FuncInfo->setLRStoreRequired(); bool isPPC64 = Subtarget.isPPC64(); auto PtrVT = getPointerTy(MF.getDataLayout()); if (Depth > 0) { SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); SDValue Offset = DAG.getConstant(Subtarget.getFrameLowering()->getReturnSaveOffset(), dl, isPPC64 ? MVT::i64 : MVT::i32); return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset), MachinePointerInfo()); } // Just load the return address off the stack. SDValue RetAddrFI = getReturnAddrFrameIndex(DAG); return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), RetAddrFI, MachinePointerInfo()); } SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); unsigned Depth = cast(Op.getOperand(0))->getZExtValue(); MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); MFI.setFrameAddressIsTaken(true); EVT PtrVT = getPointerTy(MF.getDataLayout()); bool isPPC64 = PtrVT == MVT::i64; // Naked functions never have a frame pointer, and so we use r1. For all // other functions, this decision must be delayed until during PEI. unsigned FrameReg; if (MF.getFunction().hasFnAttribute(Attribute::Naked)) FrameReg = isPPC64 ? PPC::X1 : PPC::R1; else FrameReg = isPPC64 ? PPC::FP8 : PPC::FP; SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT); while (Depth--) FrameAddr = DAG.getLoad(Op.getValueType(), dl, DAG.getEntryNode(), FrameAddr, MachinePointerInfo()); return FrameAddr; } // FIXME? Maybe this could be a TableGen attribute on some registers and // this table could be generated automatically from RegInfo. unsigned PPCTargetLowering::getRegisterByName(const char* RegName, EVT VT, SelectionDAG &DAG) const { bool isPPC64 = Subtarget.isPPC64(); bool isDarwinABI = Subtarget.isDarwinABI(); if ((isPPC64 && VT != MVT::i64 && VT != MVT::i32) || (!isPPC64 && VT != MVT::i32)) report_fatal_error("Invalid register global variable type"); bool is64Bit = isPPC64 && VT == MVT::i64; unsigned Reg = StringSwitch(RegName) .Case("r1", is64Bit ? PPC::X1 : PPC::R1) .Case("r2", (isDarwinABI || isPPC64) ? 0 : PPC::R2) .Case("r13", (!isPPC64 && isDarwinABI) ? 0 : (is64Bit ? PPC::X13 : PPC::R13)) .Default(0); if (Reg) return Reg; report_fatal_error("Invalid register name global variable"); } bool PPCTargetLowering::isAccessedAsGotIndirect(SDValue GA) const { // 32-bit SVR4 ABI access everything as got-indirect. if (Subtarget.isSVR4ABI() && !Subtarget.isPPC64()) return true; CodeModel::Model CModel = getTargetMachine().getCodeModel(); // If it is small or large code model, module locals are accessed // indirectly by loading their address from .toc/.got. The difference // is that for large code model we have ADDISTocHa + LDtocL and for // small code model we simply have LDtoc. if (CModel == CodeModel::Small || CModel == CodeModel::Large) return true; // JumpTable and BlockAddress are accessed as got-indirect. if (isa(GA) || isa(GA)) return true; if (GlobalAddressSDNode *G = dyn_cast(GA)) { const GlobalValue *GV = G->getGlobal(); unsigned char GVFlags = Subtarget.classifyGlobalReference(GV); // The NLP flag indicates that a global access has to use an // extra indirection. if (GVFlags & PPCII::MO_NLP_FLAG) return true; } return false; } bool PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { // The PowerPC target isn't yet aware of offsets. return false; } bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const { switch (Intrinsic) { case Intrinsic::ppc_qpx_qvlfd: case Intrinsic::ppc_qpx_qvlfs: case Intrinsic::ppc_qpx_qvlfcd: case Intrinsic::ppc_qpx_qvlfcs: case Intrinsic::ppc_qpx_qvlfiwa: case Intrinsic::ppc_qpx_qvlfiwz: case Intrinsic::ppc_altivec_lvx: case Intrinsic::ppc_altivec_lvxl: case Intrinsic::ppc_altivec_lvebx: case Intrinsic::ppc_altivec_lvehx: case Intrinsic::ppc_altivec_lvewx: case Intrinsic::ppc_vsx_lxvd2x: case Intrinsic::ppc_vsx_lxvw4x: { EVT VT; switch (Intrinsic) { case Intrinsic::ppc_altivec_lvebx: VT = MVT::i8; break; case Intrinsic::ppc_altivec_lvehx: VT = MVT::i16; break; case Intrinsic::ppc_altivec_lvewx: VT = MVT::i32; break; case Intrinsic::ppc_vsx_lxvd2x: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvlfd: VT = MVT::v4f64; break; case Intrinsic::ppc_qpx_qvlfs: VT = MVT::v4f32; break; case Intrinsic::ppc_qpx_qvlfcd: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvlfcs: VT = MVT::v2f32; break; default: VT = MVT::v4i32; break; } Info.opc = ISD::INTRINSIC_W_CHAIN; Info.memVT = VT; Info.ptrVal = I.getArgOperand(0); Info.offset = -VT.getStoreSize()+1; Info.size = 2*VT.getStoreSize()-1; Info.align = 1; Info.flags = MachineMemOperand::MOLoad; return true; } case Intrinsic::ppc_qpx_qvlfda: case Intrinsic::ppc_qpx_qvlfsa: case Intrinsic::ppc_qpx_qvlfcda: case Intrinsic::ppc_qpx_qvlfcsa: case Intrinsic::ppc_qpx_qvlfiwaa: case Intrinsic::ppc_qpx_qvlfiwza: { EVT VT; switch (Intrinsic) { case Intrinsic::ppc_qpx_qvlfda: VT = MVT::v4f64; break; case Intrinsic::ppc_qpx_qvlfsa: VT = MVT::v4f32; break; case Intrinsic::ppc_qpx_qvlfcda: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvlfcsa: VT = MVT::v2f32; break; default: VT = MVT::v4i32; break; } Info.opc = ISD::INTRINSIC_W_CHAIN; Info.memVT = VT; Info.ptrVal = I.getArgOperand(0); Info.offset = 0; Info.size = VT.getStoreSize(); Info.align = 1; Info.flags = MachineMemOperand::MOLoad; return true; } case Intrinsic::ppc_qpx_qvstfd: case Intrinsic::ppc_qpx_qvstfs: case Intrinsic::ppc_qpx_qvstfcd: case Intrinsic::ppc_qpx_qvstfcs: case Intrinsic::ppc_qpx_qvstfiw: case Intrinsic::ppc_altivec_stvx: case Intrinsic::ppc_altivec_stvxl: case Intrinsic::ppc_altivec_stvebx: case Intrinsic::ppc_altivec_stvehx: case Intrinsic::ppc_altivec_stvewx: case Intrinsic::ppc_vsx_stxvd2x: case Intrinsic::ppc_vsx_stxvw4x: { EVT VT; switch (Intrinsic) { case Intrinsic::ppc_altivec_stvebx: VT = MVT::i8; break; case Intrinsic::ppc_altivec_stvehx: VT = MVT::i16; break; case Intrinsic::ppc_altivec_stvewx: VT = MVT::i32; break; case Intrinsic::ppc_vsx_stxvd2x: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvstfd: VT = MVT::v4f64; break; case Intrinsic::ppc_qpx_qvstfs: VT = MVT::v4f32; break; case Intrinsic::ppc_qpx_qvstfcd: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvstfcs: VT = MVT::v2f32; break; default: VT = MVT::v4i32; break; } Info.opc = ISD::INTRINSIC_VOID; Info.memVT = VT; Info.ptrVal = I.getArgOperand(1); Info.offset = -VT.getStoreSize()+1; Info.size = 2*VT.getStoreSize()-1; Info.align = 1; Info.flags = MachineMemOperand::MOStore; return true; } case Intrinsic::ppc_qpx_qvstfda: case Intrinsic::ppc_qpx_qvstfsa: case Intrinsic::ppc_qpx_qvstfcda: case Intrinsic::ppc_qpx_qvstfcsa: case Intrinsic::ppc_qpx_qvstfiwa: { EVT VT; switch (Intrinsic) { case Intrinsic::ppc_qpx_qvstfda: VT = MVT::v4f64; break; case Intrinsic::ppc_qpx_qvstfsa: VT = MVT::v4f32; break; case Intrinsic::ppc_qpx_qvstfcda: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvstfcsa: VT = MVT::v2f32; break; default: VT = MVT::v4i32; break; } Info.opc = ISD::INTRINSIC_VOID; Info.memVT = VT; Info.ptrVal = I.getArgOperand(1); Info.offset = 0; Info.size = VT.getStoreSize(); Info.align = 1; Info.flags = MachineMemOperand::MOStore; return true; } default: break; } return false; } /// getOptimalMemOpType - Returns the target specific optimal type for load /// and store operations as a result of memset, memcpy, and memmove /// lowering. If DstAlign is zero that means it's safe to destination /// alignment can satisfy any constraint. Similarly if SrcAlign is zero it /// means there isn't a need to check it against alignment requirement, /// probably because the source does not need to be loaded. If 'IsMemset' is /// true, that means it's expanding a memset. If 'ZeroMemset' is true, that /// means it's a memset of zero. 'MemcpyStrSrc' indicates whether the memcpy /// source is constant so it does not need to be loaded. /// It returns EVT::Other if the type should be determined using generic /// target-independent logic. EVT PPCTargetLowering::getOptimalMemOpType( uint64_t Size, unsigned DstAlign, unsigned SrcAlign, bool IsMemset, bool ZeroMemset, bool MemcpyStrSrc, const AttributeList &FuncAttributes) const { if (getTargetMachine().getOptLevel() != CodeGenOpt::None) { // When expanding a memset, require at least two QPX instructions to cover // the cost of loading the value to be stored from the constant pool. if (Subtarget.hasQPX() && Size >= 32 && (!IsMemset || Size >= 64) && (!SrcAlign || SrcAlign >= 32) && (!DstAlign || DstAlign >= 32) && !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) { return MVT::v4f64; } // We should use Altivec/VSX loads and stores when available. For unaligned // addresses, unaligned VSX loads are only fast starting with the P8. if (Subtarget.hasAltivec() && Size >= 16 && (((!SrcAlign || SrcAlign >= 16) && (!DstAlign || DstAlign >= 16)) || ((IsMemset && Subtarget.hasVSX()) || Subtarget.hasP8Vector()))) return MVT::v4i32; } if (Subtarget.isPPC64()) { return MVT::i64; } return MVT::i32; } /// Returns true if it is beneficial to convert a load of a constant /// to just the constant itself. bool PPCTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const { assert(Ty->isIntegerTy()); unsigned BitSize = Ty->getPrimitiveSizeInBits(); return !(BitSize == 0 || BitSize > 64); } bool PPCTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const { if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) return false; unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); return NumBits1 == 64 && NumBits2 == 32; } bool PPCTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const { if (!VT1.isInteger() || !VT2.isInteger()) return false; unsigned NumBits1 = VT1.getSizeInBits(); unsigned NumBits2 = VT2.getSizeInBits(); return NumBits1 == 64 && NumBits2 == 32; } bool PPCTargetLowering::isZExtFree(SDValue Val, EVT VT2) const { // Generally speaking, zexts are not free, but they are free when they can be // folded with other operations. if (LoadSDNode *LD = dyn_cast(Val)) { EVT MemVT = LD->getMemoryVT(); if ((MemVT == MVT::i1 || MemVT == MVT::i8 || MemVT == MVT::i16 || (Subtarget.isPPC64() && MemVT == MVT::i32)) && (LD->getExtensionType() == ISD::NON_EXTLOAD || LD->getExtensionType() == ISD::ZEXTLOAD)) return true; } // FIXME: Add other cases... // - 32-bit shifts with a zext to i64 // - zext after ctlz, bswap, etc. // - zext after and by a constant mask return TargetLowering::isZExtFree(Val, VT2); } bool PPCTargetLowering::isFPExtFree(EVT DestVT, EVT SrcVT) const { assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() && "invalid fpext types"); // Extending to float128 is not free. if (DestVT == MVT::f128) return false; return true; } bool PPCTargetLowering::isLegalICmpImmediate(int64_t Imm) const { return isInt<16>(Imm) || isUInt<16>(Imm); } bool PPCTargetLowering::isLegalAddImmediate(int64_t Imm) const { return isInt<16>(Imm) || isUInt<16>(Imm); } bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, unsigned, MachineMemOperand::Flags, bool *Fast) const { if (DisablePPCUnaligned) return false; // PowerPC supports unaligned memory access for simple non-vector types. // Although accessing unaligned addresses is not as efficient as accessing // aligned addresses, it is generally more efficient than manual expansion, // and generally only traps for software emulation when crossing page // boundaries. if (!VT.isSimple()) return false; if (VT.getSimpleVT().isVector()) { if (Subtarget.hasVSX()) { if (VT != MVT::v2f64 && VT != MVT::v2i64 && VT != MVT::v4f32 && VT != MVT::v4i32) return false; } else { return false; } } if (VT == MVT::ppcf128) return false; if (Fast) *Fast = true; return true; } bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { VT = VT.getScalarType(); if (!VT.isSimple()) return false; switch (VT.getSimpleVT().SimpleTy) { case MVT::f32: case MVT::f64: return true; case MVT::f128: return (EnableQuadPrecision && Subtarget.hasP9Vector()); default: break; } return false; } const MCPhysReg * PPCTargetLowering::getScratchRegisters(CallingConv::ID) const { // LR is a callee-save register, but we must treat it as clobbered by any call // site. Hence we include LR in the scratch registers, which are in turn added // as implicit-defs for stackmaps and patchpoints. The same reasoning applies // to CTR, which is used by any indirect call. static const MCPhysReg ScratchRegs[] = { PPC::X12, PPC::LR8, PPC::CTR8, 0 }; return ScratchRegs; } unsigned PPCTargetLowering::getExceptionPointerRegister( const Constant *PersonalityFn) const { return Subtarget.isPPC64() ? PPC::X3 : PPC::R3; } unsigned PPCTargetLowering::getExceptionSelectorRegister( const Constant *PersonalityFn) const { return Subtarget.isPPC64() ? PPC::X4 : PPC::R4; } bool PPCTargetLowering::shouldExpandBuildVectorWithShuffles( EVT VT , unsigned DefinedValues) const { if (VT == MVT::v2i64) return Subtarget.hasDirectMove(); // Don't need stack ops with direct moves if (Subtarget.hasVSX() || Subtarget.hasQPX()) return true; return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues); } Sched::Preference PPCTargetLowering::getSchedulingPreference(SDNode *N) const { if (DisableILPPref || Subtarget.enableMachineScheduler()) return TargetLowering::getSchedulingPreference(N); return Sched::ILP; } // Create a fast isel object. FastISel * PPCTargetLowering::createFastISel(FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const { return PPC::createFastISel(FuncInfo, LibInfo); } void PPCTargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const { if (Subtarget.isDarwinABI()) return; if (!Subtarget.isPPC64()) return; // Update IsSplitCSR in PPCFunctionInfo PPCFunctionInfo *PFI = Entry->getParent()->getInfo(); PFI->setIsSplitCSR(true); } void PPCTargetLowering::insertCopiesSplitCSR( MachineBasicBlock *Entry, const SmallVectorImpl &Exits) const { const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo(); const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent()); if (!IStart) return; const TargetInstrInfo *TII = Subtarget.getInstrInfo(); MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo(); MachineBasicBlock::iterator MBBI = Entry->begin(); for (const MCPhysReg *I = IStart; *I; ++I) { const TargetRegisterClass *RC = nullptr; if (PPC::G8RCRegClass.contains(*I)) RC = &PPC::G8RCRegClass; else if (PPC::F8RCRegClass.contains(*I)) RC = &PPC::F8RCRegClass; else if (PPC::CRRCRegClass.contains(*I)) RC = &PPC::CRRCRegClass; else if (PPC::VRRCRegClass.contains(*I)) RC = &PPC::VRRCRegClass; else llvm_unreachable("Unexpected register class in CSRsViaCopy!"); unsigned NewVR = MRI->createVirtualRegister(RC); // Create copy from CSR to a virtual register. // FIXME: this currently does not emit CFI pseudo-instructions, it works // fine for CXX_FAST_TLS since the C++-style TLS access functions should be // nounwind. If we want to generalize this later, we may need to emit // CFI pseudo-instructions. assert(Entry->getParent()->getFunction().hasFnAttribute( Attribute::NoUnwind) && "Function should be nounwind in insertCopiesSplitCSR!"); Entry->addLiveIn(*I); BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR) .addReg(*I); // Insert the copy-back instructions right before the terminator. for (auto *Exit : Exits) BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(), TII->get(TargetOpcode::COPY), *I) .addReg(NewVR); } } // Override to enable LOAD_STACK_GUARD lowering on Linux. bool PPCTargetLowering::useLoadStackGuardNode() const { if (!Subtarget.isTargetLinux()) return TargetLowering::useLoadStackGuardNode(); return true; } // Override to disable global variable loading on Linux. void PPCTargetLowering::insertSSPDeclarations(Module &M) const { if (!Subtarget.isTargetLinux()) return TargetLowering::insertSSPDeclarations(M); } bool PPCTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const { if (!VT.isSimple() || !Subtarget.hasVSX()) return false; switch(VT.getSimpleVT().SimpleTy) { default: // For FP types that are currently not supported by PPC backend, return // false. Examples: f16, f80. return false; case MVT::f32: case MVT::f64: case MVT::ppcf128: return Imm.isPosZero(); } } // For vector shift operation op, fold // (op x, (and y, ((1 << numbits(x)) - 1))) -> (target op x, y) static SDValue stripModuloOnShift(const TargetLowering &TLI, SDNode *N, SelectionDAG &DAG) { SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); EVT VT = N0.getValueType(); unsigned OpSizeInBits = VT.getScalarSizeInBits(); unsigned Opcode = N->getOpcode(); unsigned TargetOpcode; switch (Opcode) { default: llvm_unreachable("Unexpected shift operation"); case ISD::SHL: TargetOpcode = PPCISD::SHL; break; case ISD::SRL: TargetOpcode = PPCISD::SRL; break; case ISD::SRA: TargetOpcode = PPCISD::SRA; break; } if (VT.isVector() && TLI.isOperationLegal(Opcode, VT) && N1->getOpcode() == ISD::AND) if (ConstantSDNode *Mask = isConstOrConstSplat(N1->getOperand(1))) if (Mask->getZExtValue() == OpSizeInBits - 1) return DAG.getNode(TargetOpcode, SDLoc(N), VT, N0, N1->getOperand(0)); return SDValue(); } SDValue PPCTargetLowering::combineSHL(SDNode *N, DAGCombinerInfo &DCI) const { if (auto Value = stripModuloOnShift(*this, N, DCI.DAG)) return Value; SDValue N0 = N->getOperand(0); ConstantSDNode *CN1 = dyn_cast(N->getOperand(1)); if (!Subtarget.isISA3_0() || N0.getOpcode() != ISD::SIGN_EXTEND || N0.getOperand(0).getValueType() != MVT::i32 || CN1 == nullptr || N->getValueType(0) != MVT::i64) return SDValue(); // We can't save an operation here if the value is already extended, and // the existing shift is easier to combine. SDValue ExtsSrc = N0.getOperand(0); if (ExtsSrc.getOpcode() == ISD::TRUNCATE && ExtsSrc.getOperand(0).getOpcode() == ISD::AssertSext) return SDValue(); SDLoc DL(N0); SDValue ShiftBy = SDValue(CN1, 0); // We want the shift amount to be i32 on the extswli, but the shift could // have an i64. if (ShiftBy.getValueType() == MVT::i64) ShiftBy = DCI.DAG.getConstant(CN1->getZExtValue(), DL, MVT::i32); return DCI.DAG.getNode(PPCISD::EXTSWSLI, DL, MVT::i64, N0->getOperand(0), ShiftBy); } SDValue PPCTargetLowering::combineSRA(SDNode *N, DAGCombinerInfo &DCI) const { if (auto Value = stripModuloOnShift(*this, N, DCI.DAG)) return Value; return SDValue(); } SDValue PPCTargetLowering::combineSRL(SDNode *N, DAGCombinerInfo &DCI) const { if (auto Value = stripModuloOnShift(*this, N, DCI.DAG)) return Value; return SDValue(); } // Transform (add X, (zext(setne Z, C))) -> (addze X, (addic (addi Z, -C), -1)) // Transform (add X, (zext(sete Z, C))) -> (addze X, (subfic (addi Z, -C), 0)) // When C is zero, the equation (addi Z, -C) can be simplified to Z // Requirement: -C in [-32768, 32767], X and Z are MVT::i64 types static SDValue combineADDToADDZE(SDNode *N, SelectionDAG &DAG, const PPCSubtarget &Subtarget) { if (!Subtarget.isPPC64()) return SDValue(); SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); auto isZextOfCompareWithConstant = [](SDValue Op) { if (Op.getOpcode() != ISD::ZERO_EXTEND || !Op.hasOneUse() || Op.getValueType() != MVT::i64) return false; SDValue Cmp = Op.getOperand(0); if (Cmp.getOpcode() != ISD::SETCC || !Cmp.hasOneUse() || Cmp.getOperand(0).getValueType() != MVT::i64) return false; if (auto *Constant = dyn_cast(Cmp.getOperand(1))) { int64_t NegConstant = 0 - Constant->getSExtValue(); // Due to the limitations of the addi instruction, // -C is required to be [-32768, 32767]. return isInt<16>(NegConstant); } return false; }; bool LHSHasPattern = isZextOfCompareWithConstant(LHS); bool RHSHasPattern = isZextOfCompareWithConstant(RHS); // If there is a pattern, canonicalize a zext operand to the RHS. if (LHSHasPattern && !RHSHasPattern) std::swap(LHS, RHS); else if (!LHSHasPattern && !RHSHasPattern) return SDValue(); SDLoc DL(N); SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Glue); SDValue Cmp = RHS.getOperand(0); SDValue Z = Cmp.getOperand(0); auto *Constant = dyn_cast(Cmp.getOperand(1)); assert(Constant && "Constant Should not be a null pointer."); int64_t NegConstant = 0 - Constant->getSExtValue(); switch(cast(Cmp.getOperand(2))->get()) { default: break; case ISD::SETNE: { // when C == 0 // --> addze X, (addic Z, -1).carry // / // add X, (zext(setne Z, C))-- // \ when -32768 <= -C <= 32767 && C != 0 // --> addze X, (addic (addi Z, -C), -1).carry SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Z, DAG.getConstant(NegConstant, DL, MVT::i64)); SDValue AddOrZ = NegConstant != 0 ? Add : Z; SDValue Addc = DAG.getNode(ISD::ADDC, DL, DAG.getVTList(MVT::i64, MVT::Glue), AddOrZ, DAG.getConstant(-1ULL, DL, MVT::i64)); return DAG.getNode(ISD::ADDE, DL, VTs, LHS, DAG.getConstant(0, DL, MVT::i64), SDValue(Addc.getNode(), 1)); } case ISD::SETEQ: { // when C == 0 // --> addze X, (subfic Z, 0).carry // / // add X, (zext(sete Z, C))-- // \ when -32768 <= -C <= 32767 && C != 0 // --> addze X, (subfic (addi Z, -C), 0).carry SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Z, DAG.getConstant(NegConstant, DL, MVT::i64)); SDValue AddOrZ = NegConstant != 0 ? Add : Z; SDValue Subc = DAG.getNode(ISD::SUBC, DL, DAG.getVTList(MVT::i64, MVT::Glue), DAG.getConstant(0, DL, MVT::i64), AddOrZ); return DAG.getNode(ISD::ADDE, DL, VTs, LHS, DAG.getConstant(0, DL, MVT::i64), SDValue(Subc.getNode(), 1)); } } return SDValue(); } SDValue PPCTargetLowering::combineADD(SDNode *N, DAGCombinerInfo &DCI) const { if (auto Value = combineADDToADDZE(N, DCI.DAG, Subtarget)) return Value; return SDValue(); } // Detect TRUNCATE operations on bitcasts of float128 values. // What we are looking for here is the situtation where we extract a subset // of bits from a 128 bit float. // This can be of two forms: // 1) BITCAST of f128 feeding TRUNCATE // 2) BITCAST of f128 feeding SRL (a shift) feeding TRUNCATE // The reason this is required is because we do not have a legal i128 type // and so we want to prevent having to store the f128 and then reload part // of it. SDValue PPCTargetLowering::combineTRUNCATE(SDNode *N, DAGCombinerInfo &DCI) const { // If we are using CRBits then try that first. if (Subtarget.useCRBits()) { // Check if CRBits did anything and return that if it did. if (SDValue CRTruncValue = DAGCombineTruncBoolExt(N, DCI)) return CRTruncValue; } SDLoc dl(N); SDValue Op0 = N->getOperand(0); // Looking for a truncate of i128 to i64. if (Op0.getValueType() != MVT::i128 || N->getValueType(0) != MVT::i64) return SDValue(); int EltToExtract = DCI.DAG.getDataLayout().isBigEndian() ? 1 : 0; // SRL feeding TRUNCATE. if (Op0.getOpcode() == ISD::SRL) { ConstantSDNode *ConstNode = dyn_cast(Op0.getOperand(1)); // The right shift has to be by 64 bits. if (!ConstNode || ConstNode->getZExtValue() != 64) return SDValue(); // Switch the element number to extract. EltToExtract = EltToExtract ? 0 : 1; // Update Op0 past the SRL. Op0 = Op0.getOperand(0); } // BITCAST feeding a TRUNCATE possibly via SRL. if (Op0.getOpcode() == ISD::BITCAST && Op0.getValueType() == MVT::i128 && Op0.getOperand(0).getValueType() == MVT::f128) { SDValue Bitcast = DCI.DAG.getBitcast(MVT::v2i64, Op0.getOperand(0)); return DCI.DAG.getNode( ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Bitcast, DCI.DAG.getTargetConstant(EltToExtract, dl, MVT::i32)); } return SDValue(); } SDValue PPCTargetLowering::combineMUL(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; ConstantSDNode *ConstOpOrElement = isConstOrConstSplat(N->getOperand(1)); if (!ConstOpOrElement) return SDValue(); // An imul is usually smaller than the alternative sequence for legal type. if (DAG.getMachineFunction().getFunction().hasMinSize() && isOperationLegal(ISD::MUL, N->getValueType(0))) return SDValue(); auto IsProfitable = [this](bool IsNeg, bool IsAddOne, EVT VT) -> bool { switch (this->Subtarget.getDarwinDirective()) { default: // TODO: enhance the condition for subtarget before pwr8 return false; case PPC::DIR_PWR8: // type mul add shl // scalar 4 1 1 // vector 7 2 2 return true; case PPC::DIR_PWR9: // type mul add shl // scalar 5 2 2 // vector 7 2 2 // The cycle RATIO of related operations are showed as a table above. // Because mul is 5(scalar)/7(vector), add/sub/shl are all 2 for both // scalar and vector type. For 2 instrs patterns, add/sub + shl // are 4, it is always profitable; but for 3 instrs patterns // (mul x, -(2^N + 1)) => -(add (shl x, N), x), sub + add + shl are 6. // So we should only do it for vector type. return IsAddOne && IsNeg ? VT.isVector() : true; } }; EVT VT = N->getValueType(0); SDLoc DL(N); const APInt &MulAmt = ConstOpOrElement->getAPIntValue(); bool IsNeg = MulAmt.isNegative(); APInt MulAmtAbs = MulAmt.abs(); if ((MulAmtAbs - 1).isPowerOf2()) { // (mul x, 2^N + 1) => (add (shl x, N), x) // (mul x, -(2^N + 1)) => -(add (shl x, N), x) if (!IsProfitable(IsNeg, true, VT)) return SDValue(); SDValue Op0 = N->getOperand(0); SDValue Op1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), DAG.getConstant((MulAmtAbs - 1).logBase2(), DL, VT)); SDValue Res = DAG.getNode(ISD::ADD, DL, VT, Op0, Op1); if (!IsNeg) return Res; return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res); } else if ((MulAmtAbs + 1).isPowerOf2()) { // (mul x, 2^N - 1) => (sub (shl x, N), x) // (mul x, -(2^N - 1)) => (sub x, (shl x, N)) if (!IsProfitable(IsNeg, false, VT)) return SDValue(); SDValue Op0 = N->getOperand(0); SDValue Op1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), DAG.getConstant((MulAmtAbs + 1).logBase2(), DL, VT)); if (!IsNeg) return DAG.getNode(ISD::SUB, DL, VT, Op1, Op0); else return DAG.getNode(ISD::SUB, DL, VT, Op0, Op1); } else { return SDValue(); } } bool PPCTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { // Only duplicate to increase tail-calls for the 64bit SysV ABIs. if (!Subtarget.isSVR4ABI() || !Subtarget.isPPC64()) return false; // If not a tail call then no need to proceed. if (!CI->isTailCall()) return false; // If tail calls are disabled for the caller then we are done. const Function *Caller = CI->getParent()->getParent(); auto Attr = Caller->getFnAttribute("disable-tail-calls"); if (Attr.getValueAsString() == "true") return false; // If sibling calls have been disabled and tail-calls aren't guaranteed // there is no reason to duplicate. auto &TM = getTargetMachine(); if (!TM.Options.GuaranteedTailCallOpt && DisableSCO) return false; // Can't tail call a function called indirectly, or if it has variadic args. const Function *Callee = CI->getCalledFunction(); if (!Callee || Callee->isVarArg()) return false; // Make sure the callee and caller calling conventions are eligible for tco. if (!areCallingConvEligibleForTCO_64SVR4(Caller->getCallingConv(), CI->getCallingConv())) return false; // If the function is local then we have a good chance at tail-calling it return getTargetMachine().shouldAssumeDSOLocal(*Caller->getParent(), Callee); } bool PPCTargetLowering::hasBitPreservingFPLogic(EVT VT) const { if (!Subtarget.hasVSX()) return false; if (Subtarget.hasP9Vector() && VT == MVT::f128) return true; return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::v4f32 || VT == MVT::v2f64; } bool PPCTargetLowering:: isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const { const Value *Mask = AndI.getOperand(1); // If the mask is suitable for andi. or andis. we should sink the and. if (const ConstantInt *CI = dyn_cast(Mask)) { // Can't handle constants wider than 64-bits. if (CI->getBitWidth() > 64) return false; int64_t ConstVal = CI->getZExtValue(); return isUInt<16>(ConstVal) || (isUInt<16>(ConstVal >> 16) && !(ConstVal & 0xFFFF)); } // For non-constant masks, we can always use the record-form and. return true; } // Transform (abs (sub (zext a), (zext b))) to (vabsd a b 0) // Transform (abs (sub (zext a), (zext_invec b))) to (vabsd a b 0) // Transform (abs (sub (zext_invec a), (zext_invec b))) to (vabsd a b 0) // Transform (abs (sub (zext_invec a), (zext b))) to (vabsd a b 0) // Transform (abs (sub a, b) to (vabsd a b 1)) if a & b of type v4i32 SDValue PPCTargetLowering::combineABS(SDNode *N, DAGCombinerInfo &DCI) const { assert((N->getOpcode() == ISD::ABS) && "Need ABS node here"); assert(Subtarget.hasP9Altivec() && "Only combine this when P9 altivec supported!"); EVT VT = N->getValueType(0); if (VT != MVT::v4i32 && VT != MVT::v8i16 && VT != MVT::v16i8) return SDValue(); SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); if (N->getOperand(0).getOpcode() == ISD::SUB) { // Even for signed integers, if it's known to be positive (as signed // integer) due to zero-extended inputs. unsigned SubOpcd0 = N->getOperand(0)->getOperand(0).getOpcode(); unsigned SubOpcd1 = N->getOperand(0)->getOperand(1).getOpcode(); if ((SubOpcd0 == ISD::ZERO_EXTEND || SubOpcd0 == ISD::ZERO_EXTEND_VECTOR_INREG) && (SubOpcd1 == ISD::ZERO_EXTEND || SubOpcd1 == ISD::ZERO_EXTEND_VECTOR_INREG)) { return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(0).getValueType(), N->getOperand(0)->getOperand(0), N->getOperand(0)->getOperand(1), DAG.getTargetConstant(0, dl, MVT::i32)); } // For type v4i32, it can be optimized with xvnegsp + vabsduw if (N->getOperand(0).getValueType() == MVT::v4i32 && N->getOperand(0).hasOneUse()) { return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(0).getValueType(), N->getOperand(0)->getOperand(0), N->getOperand(0)->getOperand(1), DAG.getTargetConstant(1, dl, MVT::i32)); } } return SDValue(); } // For type v4i32/v8ii16/v16i8, transform // from (vselect (setcc a, b, setugt), (sub a, b), (sub b, a)) to (vabsd a, b) // from (vselect (setcc a, b, setuge), (sub a, b), (sub b, a)) to (vabsd a, b) // from (vselect (setcc a, b, setult), (sub b, a), (sub a, b)) to (vabsd a, b) // from (vselect (setcc a, b, setule), (sub b, a), (sub a, b)) to (vabsd a, b) SDValue PPCTargetLowering::combineVSelect(SDNode *N, DAGCombinerInfo &DCI) const { assert((N->getOpcode() == ISD::VSELECT) && "Need VSELECT node here"); assert(Subtarget.hasP9Altivec() && "Only combine this when P9 altivec supported!"); SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); SDValue Cond = N->getOperand(0); SDValue TrueOpnd = N->getOperand(1); SDValue FalseOpnd = N->getOperand(2); EVT VT = N->getOperand(1).getValueType(); if (Cond.getOpcode() != ISD::SETCC || TrueOpnd.getOpcode() != ISD::SUB || FalseOpnd.getOpcode() != ISD::SUB) return SDValue(); // ABSD only available for type v4i32/v8i16/v16i8 if (VT != MVT::v4i32 && VT != MVT::v8i16 && VT != MVT::v16i8) return SDValue(); // At least to save one more dependent computation if (!(Cond.hasOneUse() || TrueOpnd.hasOneUse() || FalseOpnd.hasOneUse())) return SDValue(); ISD::CondCode CC = cast(Cond.getOperand(2))->get(); // Can only handle unsigned comparison here switch (CC) { default: return SDValue(); case ISD::SETUGT: case ISD::SETUGE: break; case ISD::SETULT: case ISD::SETULE: std::swap(TrueOpnd, FalseOpnd); break; } SDValue CmpOpnd1 = Cond.getOperand(0); SDValue CmpOpnd2 = Cond.getOperand(1); // SETCC CmpOpnd1 CmpOpnd2 cond // TrueOpnd = CmpOpnd1 - CmpOpnd2 // FalseOpnd = CmpOpnd2 - CmpOpnd1 if (TrueOpnd.getOperand(0) == CmpOpnd1 && TrueOpnd.getOperand(1) == CmpOpnd2 && FalseOpnd.getOperand(0) == CmpOpnd2 && FalseOpnd.getOperand(1) == CmpOpnd1) { return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(1).getValueType(), CmpOpnd1, CmpOpnd2, DAG.getTargetConstant(0, dl, MVT::i32)); } return SDValue(); } Index: stable/12/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp =================================================================== --- stable/12/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp (revision 356466) +++ stable/12/contrib/llvm-project/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp (revision 356467) @@ -1,531 +1,545 @@ //===-- PPCTargetMachine.cpp - Define TargetMachine for PowerPC -----------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // Top-level implementation for the PowerPC target. // //===----------------------------------------------------------------------===// #include "PPCTargetMachine.h" #include "MCTargetDesc/PPCMCTargetDesc.h" #include "PPC.h" #include "PPCMachineScheduler.h" #include "PPCSubtarget.h" #include "PPCTargetObjectFile.h" #include "PPCTargetTransformInfo.h" #include "TargetInfo/PowerPCTargetInfo.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/MachineScheduler.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Function.h" #include "llvm/Pass.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Target/TargetLoweringObjectFile.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Transforms/Scalar.h" #include #include #include using namespace llvm; static cl::opt EnableBranchCoalescing("enable-ppc-branch-coalesce", cl::Hidden, cl::desc("enable coalescing of duplicate branches for PPC")); static cl:: opt DisableCTRLoops("disable-ppc-ctrloops", cl::Hidden, cl::desc("Disable CTR loops for PPC")); static cl:: opt DisablePreIncPrep("disable-ppc-preinc-prep", cl::Hidden, cl::desc("Disable PPC loop preinc prep")); static cl::opt VSXFMAMutateEarly("schedule-ppc-vsx-fma-mutation-early", cl::Hidden, cl::desc("Schedule VSX FMA instruction mutation early")); static cl:: opt DisableVSXSwapRemoval("disable-ppc-vsx-swap-removal", cl::Hidden, cl::desc("Disable VSX Swap Removal for PPC")); static cl:: opt DisableQPXLoadSplat("disable-ppc-qpx-load-splat", cl::Hidden, cl::desc("Disable QPX load splat simplification")); static cl:: opt DisableMIPeephole("disable-ppc-peephole", cl::Hidden, cl::desc("Disable machine peepholes for PPC")); static cl::opt EnableGEPOpt("ppc-gep-opt", cl::Hidden, cl::desc("Enable optimizations on complex GEPs"), cl::init(true)); static cl::opt EnablePrefetch("enable-ppc-prefetching", cl::desc("disable software prefetching on PPC"), cl::init(false), cl::Hidden); static cl::opt EnableExtraTOCRegDeps("enable-ppc-extra-toc-reg-deps", cl::desc("Add extra TOC register dependencies"), cl::init(true), cl::Hidden); static cl::opt EnableMachineCombinerPass("ppc-machine-combiner", cl::desc("Enable the machine combiner pass"), cl::init(true), cl::Hidden); static cl::opt ReduceCRLogical("ppc-reduce-cr-logicals", cl::desc("Expand eligible cr-logical binary ops to branches"), cl::init(false), cl::Hidden); extern "C" void LLVMInitializePowerPCTarget() { // Register the targets RegisterTargetMachine A(getThePPC32Target()); RegisterTargetMachine B(getThePPC64Target()); RegisterTargetMachine C(getThePPC64LETarget()); PassRegistry &PR = *PassRegistry::getPassRegistry(); #ifndef NDEBUG initializePPCCTRLoopsVerifyPass(PR); #endif initializePPCLoopPreIncPrepPass(PR); initializePPCTOCRegDepsPass(PR); initializePPCEarlyReturnPass(PR); initializePPCVSXCopyPass(PR); initializePPCVSXFMAMutatePass(PR); initializePPCVSXSwapRemovalPass(PR); initializePPCReduceCRLogicalsPass(PR); initializePPCBSelPass(PR); initializePPCBranchCoalescingPass(PR); initializePPCQPXLoadSplatPass(PR); initializePPCBoolRetToIntPass(PR); initializePPCExpandISELPass(PR); initializePPCPreEmitPeepholePass(PR); initializePPCTLSDynamicCallPass(PR); initializePPCMIPeepholePass(PR); } /// Return the datalayout string of a subtarget. static std::string getDataLayoutString(const Triple &T) { bool is64Bit = T.getArch() == Triple::ppc64 || T.getArch() == Triple::ppc64le; std::string Ret; // Most PPC* platforms are big endian, PPC64LE is little endian. if (T.getArch() == Triple::ppc64le) Ret = "e"; else Ret = "E"; Ret += DataLayout::getManglingComponent(T); // PPC32 has 32 bit pointers. The PS3 (OS Lv2) is a PPC64 machine with 32 bit // pointers. if (!is64Bit || T.getOS() == Triple::Lv2) Ret += "-p:32:32"; // Note, the alignment values for f64 and i64 on ppc64 in Darwin // documentation are wrong; these are correct (i.e. "what gcc does"). if (is64Bit || !T.isOSDarwin()) Ret += "-i64:64"; else Ret += "-f64:32:64"; // PPC64 has 32 and 64 bit registers, PPC32 has only 32 bit ones. if (is64Bit) Ret += "-n32:64"; else Ret += "-n32"; return Ret; } static std::string computeFSAdditions(StringRef FS, CodeGenOpt::Level OL, const Triple &TT) { std::string FullFS = FS; // Make sure 64-bit features are available when CPUname is generic if (TT.getArch() == Triple::ppc64 || TT.getArch() == Triple::ppc64le) { if (!FullFS.empty()) FullFS = "+64bit," + FullFS; else FullFS = "+64bit"; } if (OL >= CodeGenOpt::Default) { if (!FullFS.empty()) FullFS = "+crbits," + FullFS; else FullFS = "+crbits"; } if (OL != CodeGenOpt::None) { if (!FullFS.empty()) FullFS = "+invariant-function-descriptors," + FullFS; else FullFS = "+invariant-function-descriptors"; } return FullFS; } static std::unique_ptr createTLOF(const Triple &TT) { // If it isn't a Mach-O file then it's going to be a linux ELF // object file. if (TT.isOSDarwin()) return llvm::make_unique(); return llvm::make_unique(); } static PPCTargetMachine::PPCABI computeTargetABI(const Triple &TT, const TargetOptions &Options) { if (TT.isOSDarwin()) report_fatal_error("Darwin is no longer supported for PowerPC"); if (Options.MCOptions.getABIName().startswith("elfv1")) return PPCTargetMachine::PPC_ABI_ELFv1; else if (Options.MCOptions.getABIName().startswith("elfv2")) return PPCTargetMachine::PPC_ABI_ELFv2; assert(Options.MCOptions.getABIName().empty() && "Unknown target-abi option!"); if (TT.isMacOSX()) return PPCTargetMachine::PPC_ABI_UNKNOWN; + if (TT.isOSFreeBSD()) { + switch (TT.getArch()) { + case Triple::ppc64le: + case Triple::ppc64: + if (TT.getOSMajorVersion() >= 13) + return PPCTargetMachine::PPC_ABI_ELFv2; + else + return PPCTargetMachine::PPC_ABI_ELFv1; + case Triple::ppc: + default: + return PPCTargetMachine::PPC_ABI_UNKNOWN; + } + } + switch (TT.getArch()) { case Triple::ppc64le: return PPCTargetMachine::PPC_ABI_ELFv2; case Triple::ppc64: if (TT.getEnvironment() == llvm::Triple::ELFv2) return PPCTargetMachine::PPC_ABI_ELFv2; return PPCTargetMachine::PPC_ABI_ELFv1; default: return PPCTargetMachine::PPC_ABI_UNKNOWN; } } static Reloc::Model getEffectiveRelocModel(const Triple &TT, Optional RM) { if (RM.hasValue()) return *RM; // Darwin defaults to dynamic-no-pic. if (TT.isOSDarwin()) return Reloc::DynamicNoPIC; // Big Endian PPC is PIC by default. if (TT.getArch() == Triple::ppc64) return Reloc::PIC_; // Rest are static by default. return Reloc::Static; } static CodeModel::Model getEffectivePPCCodeModel(const Triple &TT, Optional CM, bool JIT) { if (CM) { if (*CM == CodeModel::Tiny) report_fatal_error("Target does not support the tiny CodeModel", false); if (*CM == CodeModel::Kernel) report_fatal_error("Target does not support the kernel CodeModel", false); return *CM; } if (!TT.isOSDarwin() && !JIT && (TT.getArch() == Triple::ppc64 || TT.getArch() == Triple::ppc64le)) return CodeModel::Medium; return CodeModel::Small; } static ScheduleDAGInstrs *createPPCMachineScheduler(MachineSchedContext *C) { const PPCSubtarget &ST = C->MF->getSubtarget(); ScheduleDAGMILive *DAG = new ScheduleDAGMILive(C, ST.usePPCPreRASchedStrategy() ? llvm::make_unique(C) : llvm::make_unique(C)); // add DAG Mutations here. DAG->addMutation(createCopyConstrainDAGMutation(DAG->TII, DAG->TRI)); return DAG; } static ScheduleDAGInstrs *createPPCPostMachineScheduler( MachineSchedContext *C) { const PPCSubtarget &ST = C->MF->getSubtarget(); ScheduleDAGMI *DAG = new ScheduleDAGMI(C, ST.usePPCPostRASchedStrategy() ? llvm::make_unique(C) : llvm::make_unique(C), true); // add DAG Mutations here. return DAG; } // The FeatureString here is a little subtle. We are modifying the feature // string with what are (currently) non-function specific overrides as it goes // into the LLVMTargetMachine constructor and then using the stored value in the // Subtarget constructor below it. PPCTargetMachine::PPCTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, Optional RM, Optional CM, CodeGenOpt::Level OL, bool JIT) : LLVMTargetMachine(T, getDataLayoutString(TT), TT, CPU, computeFSAdditions(FS, OL, TT), Options, getEffectiveRelocModel(TT, RM), getEffectivePPCCodeModel(TT, CM, JIT), OL), TLOF(createTLOF(getTargetTriple())), TargetABI(computeTargetABI(TT, Options)) { initAsmInfo(); } PPCTargetMachine::~PPCTargetMachine() = default; const PPCSubtarget * PPCTargetMachine::getSubtargetImpl(const Function &F) const { Attribute CPUAttr = F.getFnAttribute("target-cpu"); Attribute FSAttr = F.getFnAttribute("target-features"); std::string CPU = !CPUAttr.hasAttribute(Attribute::None) ? CPUAttr.getValueAsString().str() : TargetCPU; std::string FS = !FSAttr.hasAttribute(Attribute::None) ? FSAttr.getValueAsString().str() : TargetFS; // FIXME: This is related to the code below to reset the target options, // we need to know whether or not the soft float flag is set on the // function before we can generate a subtarget. We also need to use // it as a key for the subtarget since that can be the only difference // between two functions. bool SoftFloat = F.getFnAttribute("use-soft-float").getValueAsString() == "true"; // If the soft float attribute is set on the function turn on the soft float // subtarget feature. if (SoftFloat) FS += FS.empty() ? "-hard-float" : ",-hard-float"; auto &I = SubtargetMap[CPU + FS]; if (!I) { // This needs to be done before we create a new subtarget since any // creation will depend on the TM and the code generation flags on the // function that reside in TargetOptions. resetTargetOptions(F); I = llvm::make_unique( TargetTriple, CPU, // FIXME: It would be good to have the subtarget additions here // not necessary. Anything that turns them on/off (overrides) ends // up being put at the end of the feature string, but the defaults // shouldn't require adding them. Fixing this means pulling Feature64Bit // out of most of the target cpus in the .td file and making it set only // as part of initialization via the TargetTriple. computeFSAdditions(FS, getOptLevel(), getTargetTriple()), *this); } return I.get(); } //===----------------------------------------------------------------------===// // Pass Pipeline Configuration //===----------------------------------------------------------------------===// namespace { /// PPC Code Generator Pass Configuration Options. class PPCPassConfig : public TargetPassConfig { public: PPCPassConfig(PPCTargetMachine &TM, PassManagerBase &PM) : TargetPassConfig(TM, PM) { // At any optimization level above -O0 we use the Machine Scheduler and not // the default Post RA List Scheduler. if (TM.getOptLevel() != CodeGenOpt::None) substitutePass(&PostRASchedulerID, &PostMachineSchedulerID); } PPCTargetMachine &getPPCTargetMachine() const { return getTM(); } void addIRPasses() override; bool addPreISel() override; bool addILPOpts() override; bool addInstSelector() override; void addMachineSSAOptimization() override; void addPreRegAlloc() override; void addPreSched2() override; void addPreEmitPass() override; ScheduleDAGInstrs * createMachineScheduler(MachineSchedContext *C) const override { return createPPCMachineScheduler(C); } ScheduleDAGInstrs * createPostMachineScheduler(MachineSchedContext *C) const override { return createPPCPostMachineScheduler(C); } }; } // end anonymous namespace TargetPassConfig *PPCTargetMachine::createPassConfig(PassManagerBase &PM) { return new PPCPassConfig(*this, PM); } void PPCPassConfig::addIRPasses() { if (TM->getOptLevel() != CodeGenOpt::None) addPass(createPPCBoolRetToIntPass()); addPass(createAtomicExpandPass()); // For the BG/Q (or if explicitly requested), add explicit data prefetch // intrinsics. bool UsePrefetching = TM->getTargetTriple().getVendor() == Triple::BGQ && getOptLevel() != CodeGenOpt::None; if (EnablePrefetch.getNumOccurrences() > 0) UsePrefetching = EnablePrefetch; if (UsePrefetching) addPass(createLoopDataPrefetchPass()); if (TM->getOptLevel() >= CodeGenOpt::Default && EnableGEPOpt) { // Call SeparateConstOffsetFromGEP pass to extract constants within indices // and lower a GEP with multiple indices to either arithmetic operations or // multiple GEPs with single index. addPass(createSeparateConstOffsetFromGEPPass(true)); // Call EarlyCSE pass to find and remove subexpressions in the lowered // result. addPass(createEarlyCSEPass()); // Do loop invariant code motion in case part of the lowered result is // invariant. addPass(createLICMPass()); } TargetPassConfig::addIRPasses(); } bool PPCPassConfig::addPreISel() { if (!DisablePreIncPrep && getOptLevel() != CodeGenOpt::None) addPass(createPPCLoopPreIncPrepPass(getPPCTargetMachine())); if (!DisableCTRLoops && getOptLevel() != CodeGenOpt::None) addPass(createHardwareLoopsPass()); return false; } bool PPCPassConfig::addILPOpts() { addPass(&EarlyIfConverterID); if (EnableMachineCombinerPass) addPass(&MachineCombinerID); return true; } bool PPCPassConfig::addInstSelector() { // Install an instruction selector. addPass(createPPCISelDag(getPPCTargetMachine(), getOptLevel())); #ifndef NDEBUG if (!DisableCTRLoops && getOptLevel() != CodeGenOpt::None) addPass(createPPCCTRLoopsVerify()); #endif addPass(createPPCVSXCopyPass()); return false; } void PPCPassConfig::addMachineSSAOptimization() { // PPCBranchCoalescingPass need to be done before machine sinking // since it merges empty blocks. if (EnableBranchCoalescing && getOptLevel() != CodeGenOpt::None) addPass(createPPCBranchCoalescingPass()); TargetPassConfig::addMachineSSAOptimization(); // For little endian, remove where possible the vector swap instructions // introduced at code generation to normalize vector element order. if (TM->getTargetTriple().getArch() == Triple::ppc64le && !DisableVSXSwapRemoval) addPass(createPPCVSXSwapRemovalPass()); // Reduce the number of cr-logical ops. if (ReduceCRLogical && getOptLevel() != CodeGenOpt::None) addPass(createPPCReduceCRLogicalsPass()); // Target-specific peephole cleanups performed after instruction // selection. if (!DisableMIPeephole) { addPass(createPPCMIPeepholePass()); addPass(&DeadMachineInstructionElimID); } } void PPCPassConfig::addPreRegAlloc() { if (getOptLevel() != CodeGenOpt::None) { initializePPCVSXFMAMutatePass(*PassRegistry::getPassRegistry()); insertPass(VSXFMAMutateEarly ? &RegisterCoalescerID : &MachineSchedulerID, &PPCVSXFMAMutateID); } // FIXME: We probably don't need to run these for -fPIE. if (getPPCTargetMachine().isPositionIndependent()) { // FIXME: LiveVariables should not be necessary here! // PPCTLSDynamicCallPass uses LiveIntervals which previously dependent on // LiveVariables. This (unnecessary) dependency has been removed now, // however a stage-2 clang build fails without LiveVariables computed here. addPass(&LiveVariablesID, false); addPass(createPPCTLSDynamicCallPass()); } if (EnableExtraTOCRegDeps) addPass(createPPCTOCRegDepsPass()); if (getOptLevel() != CodeGenOpt::None) addPass(&MachinePipelinerID); } void PPCPassConfig::addPreSched2() { if (getOptLevel() != CodeGenOpt::None) { addPass(&IfConverterID); // This optimization must happen after anything that might do store-to-load // forwarding. Here we're after RA (and, thus, when spills are inserted) // but before post-RA scheduling. if (!DisableQPXLoadSplat) addPass(createPPCQPXLoadSplatPass()); } } void PPCPassConfig::addPreEmitPass() { addPass(createPPCPreEmitPeepholePass()); addPass(createPPCExpandISELPass()); if (getOptLevel() != CodeGenOpt::None) addPass(createPPCEarlyReturnPass(), false); // Must run branch selection immediately preceding the asm printer. addPass(createPPCBranchSelectionPass(), false); } TargetTransformInfo PPCTargetMachine::getTargetTransformInfo(const Function &F) { return TargetTransformInfo(PPCTTIImpl(this, F)); } static MachineSchedRegistry PPCPreRASchedRegistry("ppc-prera", "Run PowerPC PreRA specific scheduler", createPPCMachineScheduler); static MachineSchedRegistry PPCPostRASchedRegistry("ppc-postra", "Run PowerPC PostRA specific scheduler", createPPCPostMachineScheduler); Index: stable/12/contrib/llvm-project/llvm =================================================================== --- stable/12/contrib/llvm-project/llvm (revision 356466) +++ stable/12/contrib/llvm-project/llvm (revision 356467) Property changes on: stable/12/contrib/llvm-project/llvm ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head/contrib/llvm-project/llvm:r356100,356104,356112 Index: stable/12/lib/libcompiler_rt/Makefile.inc =================================================================== --- stable/12/lib/libcompiler_rt/Makefile.inc (revision 356466) +++ stable/12/lib/libcompiler_rt/Makefile.inc (revision 356467) @@ -1,242 +1,250 @@ # $FreeBSD$ CRTARCH= ${MACHINE_CPUARCH:C/amd64/x86_64/} CRTSRC= ${SRCTOP}/contrib/llvm-project/compiler-rt/lib/builtins .PATH: ${CRTSRC}/${CRTARCH} .PATH: ${CRTSRC} SRCF+= absvdi2 SRCF+= absvsi2 SRCF+= absvti2 SRCF+= addvdi3 SRCF+= addvsi3 SRCF+= addvti3 SRCF+= apple_versioning SRCF+= ashldi3 SRCF+= ashlti3 SRCF+= ashrdi3 SRCF+= ashrti3 SRCF+= clear_cache SRCF+= clzdi2 SRCF+= clzsi2 SRCF+= clzti2 SRCF+= cmpdi2 SRCF+= cmpti2 SRCF+= ctzdi2 SRCF+= ctzsi2 SRCF+= ctzti2 SRCF+= divdc3 SRCF+= divdi3 SRCF+= divmoddi4 SRCF+= divmodsi4 SRCF+= divsc3 SRCF+= divsi3 SRCF+= divtc3 SRCF+= divti3 SRCF+= divxc3 SRCF+= enable_execute_stack SRCF+= eprintf SRCF+= extendhfsf2 SRCF+= ffsdi2 SRCF+= ffssi2 SRCF+= ffsti2 SRCF+= fixdfdi SRCF+= fixdfti SRCF+= fixsfdi SRCF+= fixsfti SRCF+= fixunsdfdi SRCF+= fixunsdfsi SRCF+= fixunsdfti SRCF+= fixunssfdi SRCF+= fixunssfsi SRCF+= fixunssfti SRCF+= fixunsxfdi SRCF+= fixunsxfsi SRCF+= fixunsxfti SRCF+= fixxfdi SRCF+= fixxfti SRCF+= floatditf SRCF+= floattidf SRCF+= floattisf SRCF+= floattixf SRCF+= floatunditf SRCF+= floatunsidf SRCF+= floatunsisf SRCF+= floatuntidf SRCF+= floatuntisf SRCF+= floatuntixf SRCF+= gcc_personality_v0 # not in upstream SRCF+= int_util SRCF+= lshrdi3 SRCF+= lshrti3 SRCF+= moddi3 SRCF+= modsi3 SRCF+= modti3 SRCF+= muldc3 SRCF+= muldi3 SRCF+= mulodi4 SRCF+= mulosi4 SRCF+= muloti4 SRCF+= mulsc3 SRCF+= multi3 SRCF+= mulvdi3 SRCF+= mulvsi3 SRCF+= mulvti3 SRCF+= multc3 SRCF+= mulxc3 SRCF+= negdf2 SRCF+= negdi2 SRCF+= negsf2 SRCF+= negti2 SRCF+= negvdi2 SRCF+= negvsi2 SRCF+= negvti2 SRCF+= paritydi2 SRCF+= paritysi2 SRCF+= parityti2 SRCF+= popcountdi2 SRCF+= popcountsi2 SRCF+= popcountti2 SRCF+= powidf2 SRCF+= powisf2 SRCF+= powitf2 SRCF+= powixf2 SRCF+= subvdi3 SRCF+= subvsi3 SRCF+= subvti3 SRCF+= trampoline_setup SRCF+= truncdfhf2 SRCF+= truncsfhf2 SRCF+= ucmpdi2 SRCF+= ucmpti2 SRCF+= udivdi3 SRCF+= udivmoddi4 SRCF+= udivmodsi4 SRCF+= udivmodti4 SRCF+= udivsi3 SRCF+= udivti3 SRCF+= umoddi3 SRCF+= umodsi3 SRCF+= umodti3 # Avoid using SSE2 instructions on i386, if unsupported. .if ${MACHINE_CPUARCH} == "i386" && empty(MACHINE_CPU:Msse2) SRCS+= floatdidf.c SRCS+= floatdisf.c SRCS+= floatdixf.c SRCS+= floatundidf.c SRCS+= floatundisf.c SRCS+= floatundixf.c .else SRCF+= floatdidf SRCF+= floatdisf SRCF+= floatdixf SRCF+= floatundidf SRCF+= floatundisf SRCF+= floatundixf .endif # __cpu_model support, only used on x86 .if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386" SRCF+= cpu_model .endif # # 128-bit quad precision long double support, # only used on some architectures. # .if ${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "riscv" SRCF+= addtf3 SRCF+= comparetf2 SRCF+= divtf3 SRCF+= extenddftf2 SRCF+= extendsftf2 SRCF+= fixtfdi SRCF+= fixtfsi SRCF+= fixtfti SRCF+= fixunstfdi SRCF+= fixunstfsi SRCF+= fixunstfti SRCF+= floatsitf SRCF+= floattitf SRCF+= floatunsitf SRCF+= floatuntitf SRCF+= multf3 SRCF+= subtf3 SRCF+= trunctfdf2 SRCF+= trunctfsf2 .endif # These are already shipped by libc.a on some architectures. .if ${MACHINE_CPUARCH} != "arm" && ${MACHINE_CPUARCH} != "mips" && \ ${MACHINE_CPUARCH} != "riscv" SRCF+= adddf3 SRCF+= addsf3 SRCF+= divdf3 SRCF+= divsf3 SRCF+= extendsfdf2 SRCF+= fixdfsi SRCF+= fixsfsi SRCF+= floatsidf SRCF+= floatsisf SRCF+= muldf3 SRCF+= mulsf3 SRCF+= subdf3 SRCF+= subsf3 SRCF+= truncdfsf2 .endif .if ${MACHINE_CPUARCH} != "arm" && ${MACHINE_CPUARCH} != "mips" SRCF+= comparedf2 SRCF+= comparesf2 .endif # FreeBSD-specific atomic intrinsics. .if ${MACHINE_CPUARCH} == "arm" .PATH: ${SRCTOP}/sys/arm/arm SRCF+= stdatomic CFLAGS+= -DEMIT_SYNC_ATOMICS .elif ${MACHINE_CPUARCH} == "mips" .PATH: ${SRCTOP}/sys/mips/mips SRCF+= stdatomic .endif + +.if "${COMPILER_TYPE}" == "clang" && \ + (${MACHINE_ARCH} == "powerpc" || ${MACHINE_ARCH} == "powerpcspe") +SRCS+= atomic.c +CFLAGS.atomic.c+= -Wno-atomic-alignment +.endif + + .for file in ${SRCF} .if ${MACHINE_ARCH:Marmv[67]*} && (!defined(CPUTYPE) || ${CPUTYPE:M*soft*} == "") \ && exists(${CRTSRC}/${CRTARCH}/${file}vfp.S) SRCS+= ${file}vfp.S . elif exists(${CRTSRC}/${CRTARCH}/${file}.S) SRCS+= ${file}.S . else SRCS+= ${file}.c . endif .endfor .if ${MACHINE_CPUARCH} == "arm" SRCS+= aeabi_div0.c SRCS+= aeabi_idivmod.S SRCS+= aeabi_ldivmod.S SRCS+= aeabi_memcmp.S SRCS+= aeabi_memcpy.S SRCS+= aeabi_memmove.S SRCS+= aeabi_memset.S SRCS+= aeabi_uidivmod.S SRCS+= aeabi_uldivmod.S SRCS+= bswapdi2.S SRCS+= bswapsi2.S SRCS+= switch16.S SRCS+= switch32.S SRCS+= switch8.S SRCS+= switchu8.S SRCS+= sync_synchronize.S .endif # On some archs GCC-6.3 requires bswap32 built-in. .if ${MACHINE_CPUARCH} == "mips" || ${MACHINE_CPUARCH} == "sparc64" SRCS+= bswapdi2.c SRCS+= bswapsi2.c .endif Index: stable/12 =================================================================== --- stable/12 (revision 356466) +++ stable/12 (revision 356467) Property changes on: stable/12 ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head:r356100,356104,356112