diff --git a/contrib/llvm-project/clang/include/clang/AST/ASTContext.h b/contrib/llvm-project/clang/include/clang/AST/ASTContext.h
index 63c11e237d6c..1bd5d7a6c1d7 100644
--- a/contrib/llvm-project/clang/include/clang/AST/ASTContext.h
+++ b/contrib/llvm-project/clang/include/clang/AST/ASTContext.h
@@ -1,3411 +1,3411 @@
 //===- ASTContext.h - Context to hold long-lived AST nodes ------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 /// \file
 /// Defines the clang::ASTContext interface.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_AST_ASTCONTEXT_H
 #define LLVM_CLANG_AST_ASTCONTEXT_H
 
 #include "clang/AST/ASTContextAllocate.h"
 #include "clang/AST/ASTFwd.h"
 #include "clang/AST/CanonicalType.h"
 #include "clang/AST/CommentCommandTraits.h"
 #include "clang/AST/ComparisonCategories.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclBase.h"
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/ExternalASTSource.h"
 #include "clang/AST/NestedNameSpecifier.h"
 #include "clang/AST/PrettyPrinter.h"
 #include "clang/AST/RawCommentList.h"
 #include "clang/AST/TemplateName.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/AddressSpaces.h"
 #include "clang/Basic/AttrKinds.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/Linkage.h"
 #include "clang/Basic/NoSanitizeList.h"
 #include "clang/Basic/OperatorKinds.h"
 #include "clang/Basic/PartialDiagnostic.h"
 #include "clang/Basic/ProfileList.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TargetCXXABI.h"
 #include "clang/Basic/XRayLists.h"
 #include "llvm/ADT/APSInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/TypeSize.h"
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
 #include <iterator>
 #include <memory>
 #include <string>
 #include <type_traits>
 #include <utility>
 #include <vector>
 
 namespace llvm {
 
 class APFixedPoint;
 class FixedPointSemantics;
 struct fltSemantics;
 template <typename T, unsigned N> class SmallPtrSet;
 
 } // namespace llvm
 
 namespace clang {
 
 class APValue;
 class ASTMutationListener;
 class ASTRecordLayout;
 class AtomicExpr;
 class BlockExpr;
 class BuiltinTemplateDecl;
 class CharUnits;
 class ConceptDecl;
 class CXXABI;
 class CXXConstructorDecl;
 class CXXMethodDecl;
 class CXXRecordDecl;
 class DiagnosticsEngine;
 class ParentMapContext;
 class DynTypedNodeList;
 class Expr;
 enum class FloatModeKind;
 class GlobalDecl;
 class MangleContext;
 class MangleNumberingContext;
 class MemberSpecializationInfo;
 class Module;
 struct MSGuidDeclParts;
 class ObjCCategoryDecl;
 class ObjCCategoryImplDecl;
 class ObjCContainerDecl;
 class ObjCImplDecl;
 class ObjCImplementationDecl;
 class ObjCInterfaceDecl;
 class ObjCIvarDecl;
 class ObjCMethodDecl;
 class ObjCPropertyDecl;
 class ObjCPropertyImplDecl;
 class ObjCProtocolDecl;
 class ObjCTypeParamDecl;
 class OMPTraitInfo;
 struct ParsedTargetAttr;
 class Preprocessor;
 class StoredDeclsMap;
 class TargetAttr;
 class TargetInfo;
 class TemplateDecl;
 class TemplateParameterList;
 class TemplateTemplateParmDecl;
 class TemplateTypeParmDecl;
 class UnresolvedSetIterator;
 class UsingShadowDecl;
 class VarTemplateDecl;
 class VTableContextBase;
 struct BlockVarCopyInit;
 
 namespace Builtin {
 
 class Context;
 
 } // namespace Builtin
 
 enum BuiltinTemplateKind : int;
 enum OpenCLTypeKind : uint8_t;
 
 namespace comments {
 
 class FullComment;
 
 } // namespace comments
 
 namespace interp {
 
 class Context;
 
 } // namespace interp
 
 namespace serialization {
 template <class> class AbstractTypeReader;
 } // namespace serialization
 
 enum class AlignRequirementKind {
   /// The alignment was not explicit in code.
   None,
 
   /// The alignment comes from an alignment attribute on a typedef.
   RequiredByTypedef,
 
   /// The alignment comes from an alignment attribute on a record type.
   RequiredByRecord,
 
   /// The alignment comes from an alignment attribute on a enum type.
   RequiredByEnum,
 };
 
 struct TypeInfo {
   uint64_t Width = 0;
   unsigned Align = 0;
   AlignRequirementKind AlignRequirement;
 
   TypeInfo() : AlignRequirement(AlignRequirementKind::None) {}
   TypeInfo(uint64_t Width, unsigned Align,
            AlignRequirementKind AlignRequirement)
       : Width(Width), Align(Align), AlignRequirement(AlignRequirement) {}
   bool isAlignRequired() {
     return AlignRequirement != AlignRequirementKind::None;
   }
 };
 
 struct TypeInfoChars {
   CharUnits Width;
   CharUnits Align;
   AlignRequirementKind AlignRequirement;
 
   TypeInfoChars() : AlignRequirement(AlignRequirementKind::None) {}
   TypeInfoChars(CharUnits Width, CharUnits Align,
                 AlignRequirementKind AlignRequirement)
       : Width(Width), Align(Align), AlignRequirement(AlignRequirement) {}
   bool isAlignRequired() {
     return AlignRequirement != AlignRequirementKind::None;
   }
 };
 
 /// Holds long-lived AST nodes (such as types and decls) that can be
 /// referred to throughout the semantic analysis of a file.
 class ASTContext : public RefCountedBase<ASTContext> {
   friend class NestedNameSpecifier;
 
   mutable SmallVector<Type *, 0> Types;
   mutable llvm::FoldingSet<ExtQuals> ExtQualNodes;
   mutable llvm::FoldingSet<ComplexType> ComplexTypes;
   mutable llvm::FoldingSet<PointerType> PointerTypes;
   mutable llvm::FoldingSet<AdjustedType> AdjustedTypes;
   mutable llvm::FoldingSet<BlockPointerType> BlockPointerTypes;
   mutable llvm::FoldingSet<LValueReferenceType> LValueReferenceTypes;
   mutable llvm::FoldingSet<RValueReferenceType> RValueReferenceTypes;
   mutable llvm::FoldingSet<MemberPointerType> MemberPointerTypes;
   mutable llvm::ContextualFoldingSet<ConstantArrayType, ASTContext &>
       ConstantArrayTypes;
   mutable llvm::FoldingSet<IncompleteArrayType> IncompleteArrayTypes;
   mutable std::vector<VariableArrayType*> VariableArrayTypes;
   mutable llvm::FoldingSet<DependentSizedArrayType> DependentSizedArrayTypes;
   mutable llvm::FoldingSet<DependentSizedExtVectorType>
     DependentSizedExtVectorTypes;
   mutable llvm::FoldingSet<DependentAddressSpaceType>
       DependentAddressSpaceTypes;
   mutable llvm::FoldingSet<VectorType> VectorTypes;
   mutable llvm::FoldingSet<DependentVectorType> DependentVectorTypes;
   mutable llvm::FoldingSet<ConstantMatrixType> MatrixTypes;
   mutable llvm::FoldingSet<DependentSizedMatrixType> DependentSizedMatrixTypes;
   mutable llvm::FoldingSet<FunctionNoProtoType> FunctionNoProtoTypes;
   mutable llvm::ContextualFoldingSet<FunctionProtoType, ASTContext&>
     FunctionProtoTypes;
   mutable llvm::FoldingSet<DependentTypeOfExprType> DependentTypeOfExprTypes;
   mutable llvm::FoldingSet<DependentDecltypeType> DependentDecltypeTypes;
   mutable llvm::FoldingSet<TemplateTypeParmType> TemplateTypeParmTypes;
   mutable llvm::FoldingSet<ObjCTypeParamType> ObjCTypeParamTypes;
   mutable llvm::FoldingSet<SubstTemplateTypeParmType>
     SubstTemplateTypeParmTypes;
   mutable llvm::FoldingSet<SubstTemplateTypeParmPackType>
     SubstTemplateTypeParmPackTypes;
   mutable llvm::ContextualFoldingSet<TemplateSpecializationType, ASTContext&>
     TemplateSpecializationTypes;
   mutable llvm::FoldingSet<ParenType> ParenTypes;
   mutable llvm::FoldingSet<UsingType> UsingTypes;
   mutable llvm::FoldingSet<ElaboratedType> ElaboratedTypes;
   mutable llvm::FoldingSet<DependentNameType> DependentNameTypes;
   mutable llvm::ContextualFoldingSet<DependentTemplateSpecializationType,
                                      ASTContext&>
     DependentTemplateSpecializationTypes;
   llvm::FoldingSet<PackExpansionType> PackExpansionTypes;
   mutable llvm::FoldingSet<ObjCObjectTypeImpl> ObjCObjectTypes;
   mutable llvm::FoldingSet<ObjCObjectPointerType> ObjCObjectPointerTypes;
   mutable llvm::FoldingSet<DependentUnaryTransformType>
     DependentUnaryTransformTypes;
   mutable llvm::ContextualFoldingSet<AutoType, ASTContext&> AutoTypes;
   mutable llvm::FoldingSet<DeducedTemplateSpecializationType>
     DeducedTemplateSpecializationTypes;
   mutable llvm::FoldingSet<AtomicType> AtomicTypes;
   llvm::FoldingSet<AttributedType> AttributedTypes;
   mutable llvm::FoldingSet<PipeType> PipeTypes;
   mutable llvm::FoldingSet<BitIntType> BitIntTypes;
   mutable llvm::FoldingSet<DependentBitIntType> DependentBitIntTypes;
 
   mutable llvm::FoldingSet<QualifiedTemplateName> QualifiedTemplateNames;
   mutable llvm::FoldingSet<DependentTemplateName> DependentTemplateNames;
   mutable llvm::FoldingSet<SubstTemplateTemplateParmStorage>
     SubstTemplateTemplateParms;
   mutable llvm::ContextualFoldingSet<SubstTemplateTemplateParmPackStorage,
                                      ASTContext&>
     SubstTemplateTemplateParmPacks;
 
   /// The set of nested name specifiers.
   ///
   /// This set is managed by the NestedNameSpecifier class.
   mutable llvm::FoldingSet<NestedNameSpecifier> NestedNameSpecifiers;
   mutable NestedNameSpecifier *GlobalNestedNameSpecifier = nullptr;
 
   /// A cache mapping from RecordDecls to ASTRecordLayouts.
   ///
   /// This is lazily created.  This is intentionally not serialized.
   mutable llvm::DenseMap<const RecordDecl*, const ASTRecordLayout*>
     ASTRecordLayouts;
   mutable llvm::DenseMap<const ObjCContainerDecl*, const ASTRecordLayout*>
     ObjCLayouts;
 
   /// A cache from types to size and alignment information.
   using TypeInfoMap = llvm::DenseMap<const Type *, struct TypeInfo>;
   mutable TypeInfoMap MemoizedTypeInfo;
 
   /// A cache from types to unadjusted alignment information. Only ARM and
   /// AArch64 targets need this information, keeping it separate prevents
   /// imposing overhead on TypeInfo size.
   using UnadjustedAlignMap = llvm::DenseMap<const Type *, unsigned>;
   mutable UnadjustedAlignMap MemoizedUnadjustedAlign;
 
   /// A cache mapping from CXXRecordDecls to key functions.
   llvm::DenseMap<const CXXRecordDecl*, LazyDeclPtr> KeyFunctions;
 
   /// Mapping from ObjCContainers to their ObjCImplementations.
   llvm::DenseMap<ObjCContainerDecl*, ObjCImplDecl*> ObjCImpls;
 
   /// Mapping from ObjCMethod to its duplicate declaration in the same
   /// interface.
   llvm::DenseMap<const ObjCMethodDecl*,const ObjCMethodDecl*> ObjCMethodRedecls;
 
   /// Mapping from __block VarDecls to BlockVarCopyInit.
   llvm::DenseMap<const VarDecl *, BlockVarCopyInit> BlockVarCopyInits;
 
   /// Mapping from GUIDs to the corresponding MSGuidDecl.
   mutable llvm::FoldingSet<MSGuidDecl> MSGuidDecls;
 
   /// Mapping from APValues to the corresponding TemplateParamObjects.
   mutable llvm::FoldingSet<TemplateParamObjectDecl> TemplateParamObjectDecls;
 
   /// A cache mapping a string value to a StringLiteral object with the same
   /// value.
   ///
   /// This is lazily created.  This is intentionally not serialized.
   mutable llvm::StringMap<StringLiteral *> StringLiteralCache;
 
   /// MD5 hash of CUID. It is calculated when first used and cached by this
   /// data member.
   mutable std::string CUIDHash;
 
   /// Representation of a "canonical" template template parameter that
   /// is used in canonical template names.
   class CanonicalTemplateTemplateParm : public llvm::FoldingSetNode {
     TemplateTemplateParmDecl *Parm;
 
   public:
     CanonicalTemplateTemplateParm(TemplateTemplateParmDecl *Parm)
         : Parm(Parm) {}
 
     TemplateTemplateParmDecl *getParam() const { return Parm; }
 
     void Profile(llvm::FoldingSetNodeID &ID, const ASTContext &C) {
       Profile(ID, C, Parm);
     }
 
     static void Profile(llvm::FoldingSetNodeID &ID,
                         const ASTContext &C,
                         TemplateTemplateParmDecl *Parm);
   };
   mutable llvm::ContextualFoldingSet<CanonicalTemplateTemplateParm,
                                      const ASTContext&>
     CanonTemplateTemplateParms;
 
   TemplateTemplateParmDecl *
     getCanonicalTemplateTemplateParmDecl(TemplateTemplateParmDecl *TTP) const;
 
   /// The typedef for the __int128_t type.
   mutable TypedefDecl *Int128Decl = nullptr;
 
   /// The typedef for the __uint128_t type.
   mutable TypedefDecl *UInt128Decl = nullptr;
 
   /// The typedef for the target specific predefined
   /// __builtin_va_list type.
   mutable TypedefDecl *BuiltinVaListDecl = nullptr;
 
   /// The typedef for the predefined \c __builtin_ms_va_list type.
   mutable TypedefDecl *BuiltinMSVaListDecl = nullptr;
 
   /// The typedef for the predefined \c id type.
   mutable TypedefDecl *ObjCIdDecl = nullptr;
 
   /// The typedef for the predefined \c SEL type.
   mutable TypedefDecl *ObjCSelDecl = nullptr;
 
   /// The typedef for the predefined \c Class type.
   mutable TypedefDecl *ObjCClassDecl = nullptr;
 
   /// The typedef for the predefined \c Protocol class in Objective-C.
   mutable ObjCInterfaceDecl *ObjCProtocolClassDecl = nullptr;
 
   /// The typedef for the predefined 'BOOL' type.
   mutable TypedefDecl *BOOLDecl = nullptr;
 
   // Typedefs which may be provided defining the structure of Objective-C
   // pseudo-builtins
   QualType ObjCIdRedefinitionType;
   QualType ObjCClassRedefinitionType;
   QualType ObjCSelRedefinitionType;
 
   /// The identifier 'bool'.
   mutable IdentifierInfo *BoolName = nullptr;
 
   /// The identifier 'NSObject'.
   mutable IdentifierInfo *NSObjectName = nullptr;
 
   /// The identifier 'NSCopying'.
   IdentifierInfo *NSCopyingName = nullptr;
 
   /// The identifier '__make_integer_seq'.
   mutable IdentifierInfo *MakeIntegerSeqName = nullptr;
 
   /// The identifier '__type_pack_element'.
   mutable IdentifierInfo *TypePackElementName = nullptr;
 
   QualType ObjCConstantStringType;
   mutable RecordDecl *CFConstantStringTagDecl = nullptr;
   mutable TypedefDecl *CFConstantStringTypeDecl = nullptr;
 
   mutable QualType ObjCSuperType;
 
   QualType ObjCNSStringType;
 
   /// The typedef declaration for the Objective-C "instancetype" type.
   TypedefDecl *ObjCInstanceTypeDecl = nullptr;
 
   /// The type for the C FILE type.
   TypeDecl *FILEDecl = nullptr;
 
   /// The type for the C jmp_buf type.
   TypeDecl *jmp_bufDecl = nullptr;
 
   /// The type for the C sigjmp_buf type.
   TypeDecl *sigjmp_bufDecl = nullptr;
 
   /// The type for the C ucontext_t type.
   TypeDecl *ucontext_tDecl = nullptr;
 
   /// Type for the Block descriptor for Blocks CodeGen.
   ///
   /// Since this is only used for generation of debug info, it is not
   /// serialized.
   mutable RecordDecl *BlockDescriptorType = nullptr;
 
   /// Type for the Block descriptor for Blocks CodeGen.
   ///
   /// Since this is only used for generation of debug info, it is not
   /// serialized.
   mutable RecordDecl *BlockDescriptorExtendedType = nullptr;
 
   /// Declaration for the CUDA cudaConfigureCall function.
   FunctionDecl *cudaConfigureCallDecl = nullptr;
 
   /// Keeps track of all declaration attributes.
   ///
   /// Since so few decls have attrs, we keep them in a hash map instead of
   /// wasting space in the Decl class.
   llvm::DenseMap<const Decl*, AttrVec*> DeclAttrs;
 
   /// A mapping from non-redeclarable declarations in modules that were
   /// merged with other declarations to the canonical declaration that they were
   /// merged into.
   llvm::DenseMap<Decl*, Decl*> MergedDecls;
 
   /// A mapping from a defining declaration to a list of modules (other
   /// than the owning module of the declaration) that contain merged
   /// definitions of that entity.
   llvm::DenseMap<NamedDecl*, llvm::TinyPtrVector<Module*>> MergedDefModules;
 
   /// Initializers for a module, in order. Each Decl will be either
   /// something that has a semantic effect on startup (such as a variable with
   /// a non-constant initializer), or an ImportDecl (which recursively triggers
   /// initialization of another module).
   struct PerModuleInitializers {
     llvm::SmallVector<Decl*, 4> Initializers;
     llvm::SmallVector<uint32_t, 4> LazyInitializers;
 
     void resolve(ASTContext &Ctx);
   };
   llvm::DenseMap<Module*, PerModuleInitializers*> ModuleInitializers;
 
   ASTContext &this_() { return *this; }
 
 public:
   /// A type synonym for the TemplateOrInstantiation mapping.
   using TemplateOrSpecializationInfo =
       llvm::PointerUnion<VarTemplateDecl *, MemberSpecializationInfo *>;
 
 private:
   friend class ASTDeclReader;
   friend class ASTReader;
   friend class ASTWriter;
   template <class> friend class serialization::AbstractTypeReader;
   friend class CXXRecordDecl;
   friend class IncrementalParser;
 
   /// A mapping to contain the template or declaration that
   /// a variable declaration describes or was instantiated from,
   /// respectively.
   ///
   /// For non-templates, this value will be NULL. For variable
   /// declarations that describe a variable template, this will be a
   /// pointer to a VarTemplateDecl. For static data members
   /// of class template specializations, this will be the
   /// MemberSpecializationInfo referring to the member variable that was
   /// instantiated or specialized. Thus, the mapping will keep track of
   /// the static data member templates from which static data members of
   /// class template specializations were instantiated.
   ///
   /// Given the following example:
   ///
   /// \code
   /// template<typename T>
   /// struct X {
   ///   static T value;
   /// };
   ///
   /// template<typename T>
   ///   T X<T>::value = T(17);
   ///
   /// int *x = &X<int>::value;
   /// \endcode
   ///
   /// This mapping will contain an entry that maps from the VarDecl for
   /// X<int>::value to the corresponding VarDecl for X<T>::value (within the
   /// class template X) and will be marked TSK_ImplicitInstantiation.
   llvm::DenseMap<const VarDecl *, TemplateOrSpecializationInfo>
   TemplateOrInstantiation;
 
   /// Keeps track of the declaration from which a using declaration was
   /// created during instantiation.
   ///
   /// The source and target declarations are always a UsingDecl, an
   /// UnresolvedUsingValueDecl, or an UnresolvedUsingTypenameDecl.
   ///
   /// For example:
   /// \code
   /// template<typename T>
   /// struct A {
   ///   void f();
   /// };
   ///
   /// template<typename T>
   /// struct B : A<T> {
   ///   using A<T>::f;
   /// };
   ///
   /// template struct B<int>;
   /// \endcode
   ///
   /// This mapping will contain an entry that maps from the UsingDecl in
   /// B<int> to the UnresolvedUsingDecl in B<T>.
   llvm::DenseMap<NamedDecl *, NamedDecl *> InstantiatedFromUsingDecl;
 
   /// Like InstantiatedFromUsingDecl, but for using-enum-declarations. Maps
   /// from the instantiated using-enum to the templated decl from whence it
   /// came.
   /// Note that using-enum-declarations cannot be dependent and
   /// thus will never be instantiated from an "unresolved"
   /// version thereof (as with using-declarations), so each mapping is from
   /// a (resolved) UsingEnumDecl to a (resolved) UsingEnumDecl.
   llvm::DenseMap<UsingEnumDecl *, UsingEnumDecl *>
       InstantiatedFromUsingEnumDecl;
 
   /// Simlarly maps instantiated UsingShadowDecls to their origin.
   llvm::DenseMap<UsingShadowDecl*, UsingShadowDecl*>
     InstantiatedFromUsingShadowDecl;
 
   llvm::DenseMap<FieldDecl *, FieldDecl *> InstantiatedFromUnnamedFieldDecl;
 
   /// Mapping that stores the methods overridden by a given C++
   /// member function.
   ///
   /// Since most C++ member functions aren't virtual and therefore
   /// don't override anything, we store the overridden functions in
   /// this map on the side rather than within the CXXMethodDecl structure.
   using CXXMethodVector = llvm::TinyPtrVector<const CXXMethodDecl *>;
   llvm::DenseMap<const CXXMethodDecl *, CXXMethodVector> OverriddenMethods;
 
   /// Mapping from each declaration context to its corresponding
   /// mangling numbering context (used for constructs like lambdas which
   /// need to be consistently numbered for the mangler).
   llvm::DenseMap<const DeclContext *, std::unique_ptr<MangleNumberingContext>>
       MangleNumberingContexts;
   llvm::DenseMap<const Decl *, std::unique_ptr<MangleNumberingContext>>
       ExtraMangleNumberingContexts;
 
   /// Side-table of mangling numbers for declarations which rarely
   /// need them (like static local vars).
   llvm::MapVector<const NamedDecl *, unsigned> MangleNumbers;
   llvm::MapVector<const VarDecl *, unsigned> StaticLocalNumbers;
   /// Mapping the associated device lambda mangling number if present.
   mutable llvm::DenseMap<const CXXRecordDecl *, unsigned>
       DeviceLambdaManglingNumbers;
 
   /// Mapping that stores parameterIndex values for ParmVarDecls when
   /// that value exceeds the bitfield size of ParmVarDeclBits.ParameterIndex.
   using ParameterIndexTable = llvm::DenseMap<const VarDecl *, unsigned>;
   ParameterIndexTable ParamIndices;
 
   ImportDecl *FirstLocalImport = nullptr;
   ImportDecl *LastLocalImport = nullptr;
 
   TranslationUnitDecl *TUDecl = nullptr;
   mutable ExternCContextDecl *ExternCContext = nullptr;
   mutable BuiltinTemplateDecl *MakeIntegerSeqDecl = nullptr;
   mutable BuiltinTemplateDecl *TypePackElementDecl = nullptr;
 
   /// The associated SourceManager object.
   SourceManager &SourceMgr;
 
   /// The language options used to create the AST associated with
   ///  this ASTContext object.
   LangOptions &LangOpts;
 
   /// NoSanitizeList object that is used by sanitizers to decide which
   /// entities should not be instrumented.
   std::unique_ptr<NoSanitizeList> NoSanitizeL;
 
   /// Function filtering mechanism to determine whether a given function
   /// should be imbued with the XRay "always" or "never" attributes.
   std::unique_ptr<XRayFunctionFilter> XRayFilter;
 
   /// ProfileList object that is used by the profile instrumentation
   /// to decide which entities should be instrumented.
   std::unique_ptr<ProfileList> ProfList;
 
   /// The allocator used to create AST objects.
   ///
   /// AST objects are never destructed; rather, all memory associated with the
   /// AST objects will be released when the ASTContext itself is destroyed.
   mutable llvm::BumpPtrAllocator BumpAlloc;
 
   /// Allocator for partial diagnostics.
   PartialDiagnostic::DiagStorageAllocator DiagAllocator;
 
   /// The current C++ ABI.
   std::unique_ptr<CXXABI> ABI;
   CXXABI *createCXXABI(const TargetInfo &T);
 
   /// The logical -> physical address space map.
   const LangASMap *AddrSpaceMap = nullptr;
 
   /// Address space map mangling must be used with language specific
   /// address spaces (e.g. OpenCL/CUDA)
   bool AddrSpaceMapMangling;
 
   const TargetInfo *Target = nullptr;
   const TargetInfo *AuxTarget = nullptr;
   clang::PrintingPolicy PrintingPolicy;
   std::unique_ptr<interp::Context> InterpContext;
   std::unique_ptr<ParentMapContext> ParentMapCtx;
 
   /// Keeps track of the deallocated DeclListNodes for future reuse.
   DeclListNode *ListNodeFreeList = nullptr;
 
 public:
   IdentifierTable &Idents;
   SelectorTable &Selectors;
   Builtin::Context &BuiltinInfo;
   const TranslationUnitKind TUKind;
   mutable DeclarationNameTable DeclarationNames;
   IntrusiveRefCntPtr<ExternalASTSource> ExternalSource;
   ASTMutationListener *Listener = nullptr;
 
   /// Returns the clang bytecode interpreter context.
   interp::Context &getInterpContext();
 
   struct CUDAConstantEvalContext {
     /// Do not allow wrong-sided variables in constant expressions.
     bool NoWrongSidedVars = false;
   } CUDAConstantEvalCtx;
   struct CUDAConstantEvalContextRAII {
     ASTContext &Ctx;
     CUDAConstantEvalContext SavedCtx;
     CUDAConstantEvalContextRAII(ASTContext &Ctx_, bool NoWrongSidedVars)
         : Ctx(Ctx_), SavedCtx(Ctx_.CUDAConstantEvalCtx) {
       Ctx_.CUDAConstantEvalCtx.NoWrongSidedVars = NoWrongSidedVars;
     }
     ~CUDAConstantEvalContextRAII() { Ctx.CUDAConstantEvalCtx = SavedCtx; }
   };
 
   /// Returns the dynamic AST node parent map context.
   ParentMapContext &getParentMapContext();
 
   // A traversal scope limits the parts of the AST visible to certain analyses.
   // RecursiveASTVisitor only visits specified children of TranslationUnitDecl.
   // getParents() will only observe reachable parent edges.
   //
   // The scope is defined by a set of "top-level" declarations which will be
   // visible under the TranslationUnitDecl.
   // Initially, it is the entire TU, represented by {getTranslationUnitDecl()}.
   //
   // After setTraversalScope({foo, bar}), the exposed AST looks like:
   // TranslationUnitDecl
   //  - foo
   //    - ...
   //  - bar
   //    - ...
   // All other siblings of foo and bar are pruned from the tree.
   // (However they are still accessible via TranslationUnitDecl->decls())
   //
   // Changing the scope clears the parent cache, which is expensive to rebuild.
   std::vector<Decl *> getTraversalScope() const { return TraversalScope; }
   void setTraversalScope(const std::vector<Decl *> &);
 
   /// Forwards to get node parents from the ParentMapContext. New callers should
   /// use ParentMapContext::getParents() directly.
   template <typename NodeT> DynTypedNodeList getParents(const NodeT &Node);
 
   const clang::PrintingPolicy &getPrintingPolicy() const {
     return PrintingPolicy;
   }
 
   void setPrintingPolicy(const clang::PrintingPolicy &Policy) {
     PrintingPolicy = Policy;
   }
 
   SourceManager& getSourceManager() { return SourceMgr; }
   const SourceManager& getSourceManager() const { return SourceMgr; }
 
   // Cleans up some of the data structures. This allows us to do cleanup
   // normally done in the destructor earlier. Renders much of the ASTContext
   // unusable, mostly the actual AST nodes, so should be called when we no
   // longer need access to the AST.
   void cleanup();
 
   llvm::BumpPtrAllocator &getAllocator() const {
     return BumpAlloc;
   }
 
   void *Allocate(size_t Size, unsigned Align = 8) const {
     return BumpAlloc.Allocate(Size, Align);
   }
   template <typename T> T *Allocate(size_t Num = 1) const {
     return static_cast<T *>(Allocate(Num * sizeof(T), alignof(T)));
   }
   void Deallocate(void *Ptr) const {}
 
   /// Allocates a \c DeclListNode or returns one from the \c ListNodeFreeList
   /// pool.
   DeclListNode *AllocateDeclListNode(clang::NamedDecl *ND) {
     if (DeclListNode *Alloc = ListNodeFreeList) {
       ListNodeFreeList = Alloc->Rest.dyn_cast<DeclListNode*>();
       Alloc->D = ND;
       Alloc->Rest = nullptr;
       return Alloc;
     }
     return new (*this) DeclListNode(ND);
   }
   /// Deallcates a \c DeclListNode by returning it to the \c ListNodeFreeList
   /// pool.
   void DeallocateDeclListNode(DeclListNode *N) {
     N->Rest = ListNodeFreeList;
     ListNodeFreeList = N;
   }
 
   /// Return the total amount of physical memory allocated for representing
   /// AST nodes and type information.
   size_t getASTAllocatedMemory() const {
     return BumpAlloc.getTotalMemory();
   }
 
   /// Return the total memory used for various side tables.
   size_t getSideTableAllocatedMemory() const;
 
   PartialDiagnostic::DiagStorageAllocator &getDiagAllocator() {
     return DiagAllocator;
   }
 
   const TargetInfo &getTargetInfo() const { return *Target; }
   const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
 
   /// getIntTypeForBitwidth -
   /// sets integer QualTy according to specified details:
   /// bitwidth, signed/unsigned.
   /// Returns empty type if there is no appropriate target types.
   QualType getIntTypeForBitwidth(unsigned DestWidth,
                                  unsigned Signed) const;
 
   /// getRealTypeForBitwidth -
   /// sets floating point QualTy according to specified bitwidth.
   /// Returns empty type if there is no appropriate target types.
   QualType getRealTypeForBitwidth(unsigned DestWidth,
                                   FloatModeKind ExplicitType) const;
 
   bool AtomicUsesUnsupportedLibcall(const AtomicExpr *E) const;
 
   const LangOptions& getLangOpts() const { return LangOpts; }
 
   // If this condition is false, typo correction must be performed eagerly
   // rather than delayed in many places, as it makes use of dependent types.
   // the condition is false for clang's C-only codepath, as it doesn't support
   // dependent types yet.
   bool isDependenceAllowed() const {
     return LangOpts.CPlusPlus || LangOpts.RecoveryAST;
   }
 
   const NoSanitizeList &getNoSanitizeList() const { return *NoSanitizeL; }
 
   const XRayFunctionFilter &getXRayFilter() const {
     return *XRayFilter;
   }
 
   const ProfileList &getProfileList() const { return *ProfList; }
 
   DiagnosticsEngine &getDiagnostics() const;
 
   FullSourceLoc getFullLoc(SourceLocation Loc) const {
     return FullSourceLoc(Loc,SourceMgr);
   }
 
   /// Return the C++ ABI kind that should be used. The C++ ABI can be overriden
   /// at compile time with `-fc++-abi=`. If this is not provided, we instead use
   /// the default ABI set by the target.
   TargetCXXABI::Kind getCXXABIKind() const;
 
   /// All comments in this translation unit.
   RawCommentList Comments;
 
   /// True if comments are already loaded from ExternalASTSource.
   mutable bool CommentsLoaded = false;
 
   /// Mapping from declaration to directly attached comment.
   ///
   /// Raw comments are owned by Comments list.  This mapping is populated
   /// lazily.
   mutable llvm::DenseMap<const Decl *, const RawComment *> DeclRawComments;
 
   /// Mapping from canonical declaration to the first redeclaration in chain
   /// that has a comment attached.
   ///
   /// Raw comments are owned by Comments list.  This mapping is populated
   /// lazily.
   mutable llvm::DenseMap<const Decl *, const Decl *> RedeclChainComments;
 
   /// Keeps track of redeclaration chains that don't have any comment attached.
   /// Mapping from canonical declaration to redeclaration chain that has no
   /// comments attached to any redeclaration. Specifically it's mapping to
   /// the last redeclaration we've checked.
   ///
   /// Shall not contain declarations that have comments attached to any
   /// redeclaration in their chain.
   mutable llvm::DenseMap<const Decl *, const Decl *> CommentlessRedeclChains;
 
   /// Mapping from declarations to parsed comments attached to any
   /// redeclaration.
   mutable llvm::DenseMap<const Decl *, comments::FullComment *> ParsedComments;
 
   /// Attaches \p Comment to \p OriginalD and to its redeclaration chain
   /// and removes the redeclaration chain from the set of commentless chains.
   ///
   /// Don't do anything if a comment has already been attached to \p OriginalD
   /// or its redeclaration chain.
   void cacheRawCommentForDecl(const Decl &OriginalD,
                               const RawComment &Comment) const;
 
   /// \returns searches \p CommentsInFile for doc comment for \p D.
   ///
   /// \p RepresentativeLocForDecl is used as a location for searching doc
   /// comments. \p CommentsInFile is a mapping offset -> comment of files in the
   /// same file where \p RepresentativeLocForDecl is.
   RawComment *getRawCommentForDeclNoCacheImpl(
       const Decl *D, const SourceLocation RepresentativeLocForDecl,
       const std::map<unsigned, RawComment *> &CommentsInFile) const;
 
   /// Return the documentation comment attached to a given declaration,
   /// without looking into cache.
   RawComment *getRawCommentForDeclNoCache(const Decl *D) const;
 
 public:
   void addComment(const RawComment &RC);
 
   /// Return the documentation comment attached to a given declaration.
   /// Returns nullptr if no comment is attached.
   ///
   /// \param OriginalDecl if not nullptr, is set to declaration AST node that
   /// had the comment, if the comment we found comes from a redeclaration.
   const RawComment *
   getRawCommentForAnyRedecl(const Decl *D,
                             const Decl **OriginalDecl = nullptr) const;
 
   /// Searches existing comments for doc comments that should be attached to \p
   /// Decls. If any doc comment is found, it is parsed.
   ///
   /// Requirement: All \p Decls are in the same file.
   ///
   /// If the last comment in the file is already attached we assume
   /// there are not comments left to be attached to \p Decls.
   void attachCommentsToJustParsedDecls(ArrayRef<Decl *> Decls,
                                        const Preprocessor *PP);
 
   /// Return parsed documentation comment attached to a given declaration.
   /// Returns nullptr if no comment is attached.
   ///
   /// \param PP the Preprocessor used with this TU.  Could be nullptr if
   /// preprocessor is not available.
   comments::FullComment *getCommentForDecl(const Decl *D,
                                            const Preprocessor *PP) const;
 
   /// Return parsed documentation comment attached to a given declaration.
   /// Returns nullptr if no comment is attached. Does not look at any
   /// redeclarations of the declaration.
   comments::FullComment *getLocalCommentForDeclUncached(const Decl *D) const;
 
   comments::FullComment *cloneFullComment(comments::FullComment *FC,
                                          const Decl *D) const;
 
 private:
   mutable comments::CommandTraits CommentCommandTraits;
 
   /// Iterator that visits import declarations.
   class import_iterator {
     ImportDecl *Import = nullptr;
 
   public:
     using value_type = ImportDecl *;
     using reference = ImportDecl *;
     using pointer = ImportDecl *;
     using difference_type = int;
     using iterator_category = std::forward_iterator_tag;
 
     import_iterator() = default;
     explicit import_iterator(ImportDecl *Import) : Import(Import) {}
 
     reference operator*() const { return Import; }
     pointer operator->() const { return Import; }
 
     import_iterator &operator++() {
       Import = ASTContext::getNextLocalImport(Import);
       return *this;
     }
 
     import_iterator operator++(int) {
       import_iterator Other(*this);
       ++(*this);
       return Other;
     }
 
     friend bool operator==(import_iterator X, import_iterator Y) {
       return X.Import == Y.Import;
     }
 
     friend bool operator!=(import_iterator X, import_iterator Y) {
       return X.Import != Y.Import;
     }
   };
 
 public:
   comments::CommandTraits &getCommentCommandTraits() const {
     return CommentCommandTraits;
   }
 
   /// Retrieve the attributes for the given declaration.
   AttrVec& getDeclAttrs(const Decl *D);
 
   /// Erase the attributes corresponding to the given declaration.
   void eraseDeclAttrs(const Decl *D);
 
   /// If this variable is an instantiated static data member of a
   /// class template specialization, returns the templated static data member
   /// from which it was instantiated.
   // FIXME: Remove ?
   MemberSpecializationInfo *getInstantiatedFromStaticDataMember(
                                                            const VarDecl *Var);
 
   /// Note that the static data member \p Inst is an instantiation of
   /// the static data member template \p Tmpl of a class template.
   void setInstantiatedFromStaticDataMember(VarDecl *Inst, VarDecl *Tmpl,
                                            TemplateSpecializationKind TSK,
                         SourceLocation PointOfInstantiation = SourceLocation());
 
   TemplateOrSpecializationInfo
   getTemplateOrSpecializationInfo(const VarDecl *Var);
 
   void setTemplateOrSpecializationInfo(VarDecl *Inst,
                                        TemplateOrSpecializationInfo TSI);
 
   /// If the given using decl \p Inst is an instantiation of
   /// another (possibly unresolved) using decl, return it.
   NamedDecl *getInstantiatedFromUsingDecl(NamedDecl *Inst);
 
   /// Remember that the using decl \p Inst is an instantiation
   /// of the using decl \p Pattern of a class template.
   void setInstantiatedFromUsingDecl(NamedDecl *Inst, NamedDecl *Pattern);
 
   /// If the given using-enum decl \p Inst is an instantiation of
   /// another using-enum decl, return it.
   UsingEnumDecl *getInstantiatedFromUsingEnumDecl(UsingEnumDecl *Inst);
 
   /// Remember that the using enum decl \p Inst is an instantiation
   /// of the using enum decl \p Pattern of a class template.
   void setInstantiatedFromUsingEnumDecl(UsingEnumDecl *Inst,
                                         UsingEnumDecl *Pattern);
 
   UsingShadowDecl *getInstantiatedFromUsingShadowDecl(UsingShadowDecl *Inst);
   void setInstantiatedFromUsingShadowDecl(UsingShadowDecl *Inst,
                                           UsingShadowDecl *Pattern);
 
   FieldDecl *getInstantiatedFromUnnamedFieldDecl(FieldDecl *Field);
 
   void setInstantiatedFromUnnamedFieldDecl(FieldDecl *Inst, FieldDecl *Tmpl);
 
   // Access to the set of methods overridden by the given C++ method.
   using overridden_cxx_method_iterator = CXXMethodVector::const_iterator;
   overridden_cxx_method_iterator
   overridden_methods_begin(const CXXMethodDecl *Method) const;
 
   overridden_cxx_method_iterator
   overridden_methods_end(const CXXMethodDecl *Method) const;
 
   unsigned overridden_methods_size(const CXXMethodDecl *Method) const;
 
   using overridden_method_range =
       llvm::iterator_range<overridden_cxx_method_iterator>;
 
   overridden_method_range overridden_methods(const CXXMethodDecl *Method) const;
 
   /// Note that the given C++ \p Method overrides the given \p
   /// Overridden method.
   void addOverriddenMethod(const CXXMethodDecl *Method,
                            const CXXMethodDecl *Overridden);
 
   /// Return C++ or ObjC overridden methods for the given \p Method.
   ///
   /// An ObjC method is considered to override any method in the class's
   /// base classes, its protocols, or its categories' protocols, that has
   /// the same selector and is of the same kind (class or instance).
   /// A method in an implementation is not considered as overriding the same
   /// method in the interface or its categories.
   void getOverriddenMethods(
                         const NamedDecl *Method,
                         SmallVectorImpl<const NamedDecl *> &Overridden) const;
 
   /// Notify the AST context that a new import declaration has been
   /// parsed or implicitly created within this translation unit.
   void addedLocalImportDecl(ImportDecl *Import);
 
   static ImportDecl *getNextLocalImport(ImportDecl *Import) {
     return Import->getNextLocalImport();
   }
 
   using import_range = llvm::iterator_range<import_iterator>;
 
   import_range local_imports() const {
     return import_range(import_iterator(FirstLocalImport), import_iterator());
   }
 
   Decl *getPrimaryMergedDecl(Decl *D) {
     Decl *Result = MergedDecls.lookup(D);
     return Result ? Result : D;
   }
   void setPrimaryMergedDecl(Decl *D, Decl *Primary) {
     MergedDecls[D] = Primary;
   }
 
   /// Note that the definition \p ND has been merged into module \p M,
   /// and should be visible whenever \p M is visible.
   void mergeDefinitionIntoModule(NamedDecl *ND, Module *M,
                                  bool NotifyListeners = true);
 
   /// Clean up the merged definition list. Call this if you might have
   /// added duplicates into the list.
   void deduplicateMergedDefinitonsFor(NamedDecl *ND);
 
   /// Get the additional modules in which the definition \p Def has
   /// been merged.
   ArrayRef<Module*> getModulesWithMergedDefinition(const NamedDecl *Def);
 
   /// Add a declaration to the list of declarations that are initialized
   /// for a module. This will typically be a global variable (with internal
   /// linkage) that runs module initializers, such as the iostream initializer,
   /// or an ImportDecl nominating another module that has initializers.
   void addModuleInitializer(Module *M, Decl *Init);
 
   void addLazyModuleInitializers(Module *M, ArrayRef<uint32_t> IDs);
 
   /// Get the initializations to perform when importing a module, if any.
   ArrayRef<Decl*> getModuleInitializers(Module *M);
 
   TranslationUnitDecl *getTranslationUnitDecl() const {
     return TUDecl->getMostRecentDecl();
   }
   void addTranslationUnitDecl() {
     assert(!TUDecl || TUKind == TU_Incremental);
     TranslationUnitDecl *NewTUDecl = TranslationUnitDecl::Create(*this);
     if (TraversalScope.empty() || TraversalScope.back() == TUDecl)
       TraversalScope = {NewTUDecl};
     if (TUDecl)
       NewTUDecl->setPreviousDecl(TUDecl);
     TUDecl = NewTUDecl;
   }
 
   ExternCContextDecl *getExternCContextDecl() const;
   BuiltinTemplateDecl *getMakeIntegerSeqDecl() const;
   BuiltinTemplateDecl *getTypePackElementDecl() const;
 
   // Builtin Types.
   CanQualType VoidTy;
   CanQualType BoolTy;
   CanQualType CharTy;
   CanQualType WCharTy;  // [C++ 3.9.1p5].
   CanQualType WideCharTy; // Same as WCharTy in C++, integer type in C99.
   CanQualType WIntTy;   // [C99 7.24.1], integer type unchanged by default promotions.
   CanQualType Char8Ty;  // [C++20 proposal]
   CanQualType Char16Ty; // [C++0x 3.9.1p5], integer type in C99.
   CanQualType Char32Ty; // [C++0x 3.9.1p5], integer type in C99.
   CanQualType SignedCharTy, ShortTy, IntTy, LongTy, LongLongTy, Int128Ty;
   CanQualType UnsignedCharTy, UnsignedShortTy, UnsignedIntTy, UnsignedLongTy;
   CanQualType UnsignedLongLongTy, UnsignedInt128Ty;
   CanQualType FloatTy, DoubleTy, LongDoubleTy, Float128Ty, Ibm128Ty;
   CanQualType ShortAccumTy, AccumTy,
       LongAccumTy;  // ISO/IEC JTC1 SC22 WG14 N1169 Extension
   CanQualType UnsignedShortAccumTy, UnsignedAccumTy, UnsignedLongAccumTy;
   CanQualType ShortFractTy, FractTy, LongFractTy;
   CanQualType UnsignedShortFractTy, UnsignedFractTy, UnsignedLongFractTy;
   CanQualType SatShortAccumTy, SatAccumTy, SatLongAccumTy;
   CanQualType SatUnsignedShortAccumTy, SatUnsignedAccumTy,
       SatUnsignedLongAccumTy;
   CanQualType SatShortFractTy, SatFractTy, SatLongFractTy;
   CanQualType SatUnsignedShortFractTy, SatUnsignedFractTy,
       SatUnsignedLongFractTy;
   CanQualType HalfTy; // [OpenCL 6.1.1.1], ARM NEON
   CanQualType BFloat16Ty;
   CanQualType Float16Ty; // C11 extension ISO/IEC TS 18661-3
   CanQualType VoidPtrTy, NullPtrTy;
   CanQualType DependentTy, OverloadTy, BoundMemberTy, UnknownAnyTy;
   CanQualType BuiltinFnTy;
   CanQualType PseudoObjectTy, ARCUnbridgedCastTy;
   CanQualType ObjCBuiltinIdTy, ObjCBuiltinClassTy, ObjCBuiltinSelTy;
   CanQualType ObjCBuiltinBoolTy;
 #define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
   CanQualType SingletonId;
 #include "clang/Basic/OpenCLImageTypes.def"
   CanQualType OCLSamplerTy, OCLEventTy, OCLClkEventTy;
   CanQualType OCLQueueTy, OCLReserveIDTy;
   CanQualType IncompleteMatrixIdxTy;
   CanQualType OMPArraySectionTy, OMPArrayShapingTy, OMPIteratorTy;
 #define EXT_OPAQUE_TYPE(ExtType, Id, Ext) \
   CanQualType Id##Ty;
 #include "clang/Basic/OpenCLExtensionTypes.def"
 #define SVE_TYPE(Name, Id, SingletonId) \
   CanQualType SingletonId;
 #include "clang/Basic/AArch64SVEACLETypes.def"
 #define PPC_VECTOR_TYPE(Name, Id, Size) \
   CanQualType Id##Ty;
 #include "clang/Basic/PPCTypes.def"
 #define RVV_TYPE(Name, Id, SingletonId) \
   CanQualType SingletonId;
 #include "clang/Basic/RISCVVTypes.def"
 
   // Types for deductions in C++0x [stmt.ranged]'s desugaring. Built on demand.
   mutable QualType AutoDeductTy;     // Deduction against 'auto'.
   mutable QualType AutoRRefDeductTy; // Deduction against 'auto &&'.
 
   // Decl used to help define __builtin_va_list for some targets.
   // The decl is built when constructing 'BuiltinVaListDecl'.
   mutable Decl *VaListTagDecl = nullptr;
 
   // Implicitly-declared type 'struct _GUID'.
   mutable TagDecl *MSGuidTagDecl = nullptr;
 
   /// Keep track of CUDA/HIP device-side variables ODR-used by host code.
   llvm::DenseSet<const VarDecl *> CUDADeviceVarODRUsedByHost;
 
   ASTContext(LangOptions &LOpts, SourceManager &SM, IdentifierTable &idents,
              SelectorTable &sels, Builtin::Context &builtins,
              TranslationUnitKind TUKind);
   ASTContext(const ASTContext &) = delete;
   ASTContext &operator=(const ASTContext &) = delete;
   ~ASTContext();
 
   /// Attach an external AST source to the AST context.
   ///
   /// The external AST source provides the ability to load parts of
   /// the abstract syntax tree as needed from some external storage,
   /// e.g., a precompiled header.
   void setExternalSource(IntrusiveRefCntPtr<ExternalASTSource> Source);
 
   /// Retrieve a pointer to the external AST source associated
   /// with this AST context, if any.
   ExternalASTSource *getExternalSource() const {
     return ExternalSource.get();
   }
 
   /// Attach an AST mutation listener to the AST context.
   ///
   /// The AST mutation listener provides the ability to track modifications to
   /// the abstract syntax tree entities committed after they were initially
   /// created.
   void setASTMutationListener(ASTMutationListener *Listener) {
     this->Listener = Listener;
   }
 
   /// Retrieve a pointer to the AST mutation listener associated
   /// with this AST context, if any.
   ASTMutationListener *getASTMutationListener() const { return Listener; }
 
   void PrintStats() const;
   const SmallVectorImpl<Type *>& getTypes() const { return Types; }
 
   BuiltinTemplateDecl *buildBuiltinTemplateDecl(BuiltinTemplateKind BTK,
                                                 const IdentifierInfo *II) const;
 
   /// Create a new implicit TU-level CXXRecordDecl or RecordDecl
   /// declaration.
   RecordDecl *buildImplicitRecord(StringRef Name,
                                   RecordDecl::TagKind TK = TTK_Struct) const;
 
   /// Create a new implicit TU-level typedef declaration.
   TypedefDecl *buildImplicitTypedef(QualType T, StringRef Name) const;
 
   /// Retrieve the declaration for the 128-bit signed integer type.
   TypedefDecl *getInt128Decl() const;
 
   /// Retrieve the declaration for the 128-bit unsigned integer type.
   TypedefDecl *getUInt128Decl() const;
 
   //===--------------------------------------------------------------------===//
   //                           Type Constructors
   //===--------------------------------------------------------------------===//
 
 private:
   /// Return a type with extended qualifiers.
   QualType getExtQualType(const Type *Base, Qualifiers Quals) const;
 
   QualType getTypeDeclTypeSlow(const TypeDecl *Decl) const;
 
   QualType getPipeType(QualType T, bool ReadOnly) const;
 
 public:
   /// Return the uniqued reference to the type for an address space
   /// qualified type with the specified type and address space.
   ///
   /// The resulting type has a union of the qualifiers from T and the address
   /// space. If T already has an address space specifier, it is silently
   /// replaced.
   QualType getAddrSpaceQualType(QualType T, LangAS AddressSpace) const;
 
   /// Remove any existing address space on the type and returns the type
   /// with qualifiers intact (or that's the idea anyway)
   ///
   /// The return type should be T with all prior qualifiers minus the address
   /// space.
   QualType removeAddrSpaceQualType(QualType T) const;
 
   /// Apply Objective-C protocol qualifiers to the given type.
   /// \param allowOnPointerType specifies if we can apply protocol
   /// qualifiers on ObjCObjectPointerType. It can be set to true when
   /// constructing the canonical type of a Objective-C type parameter.
   QualType applyObjCProtocolQualifiers(QualType type,
       ArrayRef<ObjCProtocolDecl *> protocols, bool &hasError,
       bool allowOnPointerType = false) const;
 
   /// Return the uniqued reference to the type for an Objective-C
   /// gc-qualified type.
   ///
   /// The resulting type has a union of the qualifiers from T and the gc
   /// attribute.
   QualType getObjCGCQualType(QualType T, Qualifiers::GC gcAttr) const;
 
   /// Remove the existing address space on the type if it is a pointer size
   /// address space and return the type with qualifiers intact.
   QualType removePtrSizeAddrSpace(QualType T) const;
 
   /// Return the uniqued reference to the type for a \c restrict
   /// qualified type.
   ///
   /// The resulting type has a union of the qualifiers from \p T and
   /// \c restrict.
   QualType getRestrictType(QualType T) const {
     return T.withFastQualifiers(Qualifiers::Restrict);
   }
 
   /// Return the uniqued reference to the type for a \c volatile
   /// qualified type.
   ///
   /// The resulting type has a union of the qualifiers from \p T and
   /// \c volatile.
   QualType getVolatileType(QualType T) const {
     return T.withFastQualifiers(Qualifiers::Volatile);
   }
 
   /// Return the uniqued reference to the type for a \c const
   /// qualified type.
   ///
   /// The resulting type has a union of the qualifiers from \p T and \c const.
   ///
   /// It can be reasonably expected that this will always be equivalent to
   /// calling T.withConst().
   QualType getConstType(QualType T) const { return T.withConst(); }
 
   /// Change the ExtInfo on a function type.
   const FunctionType *adjustFunctionType(const FunctionType *Fn,
                                          FunctionType::ExtInfo EInfo);
 
   /// Adjust the given function result type.
   CanQualType getCanonicalFunctionResultType(QualType ResultType) const;
 
   /// Change the result type of a function type once it is deduced.
   void adjustDeducedFunctionResultType(FunctionDecl *FD, QualType ResultType);
 
   /// Get a function type and produce the equivalent function type with the
   /// specified exception specification. Type sugar that can be present on a
   /// declaration of a function with an exception specification is permitted
   /// and preserved. Other type sugar (for instance, typedefs) is not.
   QualType getFunctionTypeWithExceptionSpec(
       QualType Orig, const FunctionProtoType::ExceptionSpecInfo &ESI);
 
   /// Determine whether two function types are the same, ignoring
   /// exception specifications in cases where they're part of the type.
   bool hasSameFunctionTypeIgnoringExceptionSpec(QualType T, QualType U);
 
   /// Change the exception specification on a function once it is
   /// delay-parsed, instantiated, or computed.
   void adjustExceptionSpec(FunctionDecl *FD,
                            const FunctionProtoType::ExceptionSpecInfo &ESI,
                            bool AsWritten = false);
 
   /// Get a function type and produce the equivalent function type where
   /// pointer size address spaces in the return type and parameter tyeps are
   /// replaced with the default address space.
   QualType getFunctionTypeWithoutPtrSizes(QualType T);
 
   /// Determine whether two function types are the same, ignoring pointer sizes
   /// in the return type and parameter types.
   bool hasSameFunctionTypeIgnoringPtrSizes(QualType T, QualType U);
 
   /// Return the uniqued reference to the type for a complex
   /// number with the specified element type.
   QualType getComplexType(QualType T) const;
   CanQualType getComplexType(CanQualType T) const {
     return CanQualType::CreateUnsafe(getComplexType((QualType) T));
   }
 
   /// Return the uniqued reference to the type for a pointer to
   /// the specified type.
   QualType getPointerType(QualType T) const;
   CanQualType getPointerType(CanQualType T) const {
     return CanQualType::CreateUnsafe(getPointerType((QualType) T));
   }
 
   /// Return the uniqued reference to a type adjusted from the original
   /// type to a new type.
   QualType getAdjustedType(QualType Orig, QualType New) const;
   CanQualType getAdjustedType(CanQualType Orig, CanQualType New) const {
     return CanQualType::CreateUnsafe(
         getAdjustedType((QualType)Orig, (QualType)New));
   }
 
   /// Return the uniqued reference to the decayed version of the given
   /// type.  Can only be called on array and function types which decay to
   /// pointer types.
   QualType getDecayedType(QualType T) const;
   CanQualType getDecayedType(CanQualType T) const {
     return CanQualType::CreateUnsafe(getDecayedType((QualType) T));
   }
 
   /// Return the uniqued reference to the atomic type for the specified
   /// type.
   QualType getAtomicType(QualType T) const;
 
   /// Return the uniqued reference to the type for a block of the
   /// specified type.
   QualType getBlockPointerType(QualType T) const;
 
   /// Gets the struct used to keep track of the descriptor for pointer to
   /// blocks.
   QualType getBlockDescriptorType() const;
 
   /// Return a read_only pipe type for the specified type.
   QualType getReadPipeType(QualType T) const;
 
   /// Return a write_only pipe type for the specified type.
   QualType getWritePipeType(QualType T) const;
 
   /// Return a bit-precise integer type with the specified signedness and bit
   /// count.
   QualType getBitIntType(bool Unsigned, unsigned NumBits) const;
 
   /// Return a dependent bit-precise integer type with the specified signedness
   /// and bit count.
   QualType getDependentBitIntType(bool Unsigned, Expr *BitsExpr) const;
 
   /// Gets the struct used to keep track of the extended descriptor for
   /// pointer to blocks.
   QualType getBlockDescriptorExtendedType() const;
 
   /// Map an AST Type to an OpenCLTypeKind enum value.
   OpenCLTypeKind getOpenCLTypeKind(const Type *T) const;
 
   /// Get address space for OpenCL type.
   LangAS getOpenCLTypeAddrSpace(const Type *T) const;
 
   /// Returns default address space based on OpenCL version and enabled features
   inline LangAS getDefaultOpenCLPointeeAddrSpace() {
     return LangOpts.OpenCLGenericAddressSpace ? LangAS::opencl_generic
                                               : LangAS::opencl_private;
   }
 
   void setcudaConfigureCallDecl(FunctionDecl *FD) {
     cudaConfigureCallDecl = FD;
   }
 
   FunctionDecl *getcudaConfigureCallDecl() {
     return cudaConfigureCallDecl;
   }
 
   /// Returns true iff we need copy/dispose helpers for the given type.
   bool BlockRequiresCopying(QualType Ty, const VarDecl *D);
 
   /// Returns true, if given type has a known lifetime. HasByrefExtendedLayout
   /// is set to false in this case. If HasByrefExtendedLayout returns true,
   /// byref variable has extended lifetime.
   bool getByrefLifetime(QualType Ty,
                         Qualifiers::ObjCLifetime &Lifetime,
                         bool &HasByrefExtendedLayout) const;
 
   /// Return the uniqued reference to the type for an lvalue reference
   /// to the specified type.
   QualType getLValueReferenceType(QualType T, bool SpelledAsLValue = true)
     const;
 
   /// Return the uniqued reference to the type for an rvalue reference
   /// to the specified type.
   QualType getRValueReferenceType(QualType T) const;
 
   /// Return the uniqued reference to the type for a member pointer to
   /// the specified type in the specified class.
   ///
   /// The class \p Cls is a \c Type because it could be a dependent name.
   QualType getMemberPointerType(QualType T, const Type *Cls) const;
 
   /// Return a non-unique reference to the type for a variable array of
   /// the specified element type.
   QualType getVariableArrayType(QualType EltTy, Expr *NumElts,
                                 ArrayType::ArraySizeModifier ASM,
                                 unsigned IndexTypeQuals,
                                 SourceRange Brackets) const;
 
   /// Return a non-unique reference to the type for a dependently-sized
   /// array of the specified element type.
   ///
   /// FIXME: We will need these to be uniqued, or at least comparable, at some
   /// point.
   QualType getDependentSizedArrayType(QualType EltTy, Expr *NumElts,
                                       ArrayType::ArraySizeModifier ASM,
                                       unsigned IndexTypeQuals,
                                       SourceRange Brackets) const;
 
   /// Return a unique reference to the type for an incomplete array of
   /// the specified element type.
   QualType getIncompleteArrayType(QualType EltTy,
                                   ArrayType::ArraySizeModifier ASM,
                                   unsigned IndexTypeQuals) const;
 
   /// Return the unique reference to the type for a constant array of
   /// the specified element type.
   QualType getConstantArrayType(QualType EltTy, const llvm::APInt &ArySize,
                                 const Expr *SizeExpr,
                                 ArrayType::ArraySizeModifier ASM,
                                 unsigned IndexTypeQuals) const;
 
   /// Return a type for a constant array for a string literal of the
   /// specified element type and length.
   QualType getStringLiteralArrayType(QualType EltTy, unsigned Length) const;
 
   /// Returns a vla type where known sizes are replaced with [*].
   QualType getVariableArrayDecayedType(QualType Ty) const;
 
   // Convenience struct to return information about a builtin vector type.
   struct BuiltinVectorTypeInfo {
     QualType ElementType;
     llvm::ElementCount EC;
     unsigned NumVectors;
     BuiltinVectorTypeInfo(QualType ElementType, llvm::ElementCount EC,
                           unsigned NumVectors)
         : ElementType(ElementType), EC(EC), NumVectors(NumVectors) {}
   };
 
   /// Returns the element type, element count and number of vectors
   /// (in case of tuple) for a builtin vector type.
   BuiltinVectorTypeInfo
   getBuiltinVectorTypeInfo(const BuiltinType *VecTy) const;
 
   /// Return the unique reference to a scalable vector type of the specified
   /// element type and scalable number of elements.
   ///
   /// \pre \p EltTy must be a built-in type.
   QualType getScalableVectorType(QualType EltTy, unsigned NumElts) const;
 
   /// Return the unique reference to a vector type of the specified
   /// element type and size.
   ///
   /// \pre \p VectorType must be a built-in type.
   QualType getVectorType(QualType VectorType, unsigned NumElts,
                          VectorType::VectorKind VecKind) const;
   /// Return the unique reference to the type for a dependently sized vector of
   /// the specified element type.
   QualType getDependentVectorType(QualType VectorType, Expr *SizeExpr,
                                   SourceLocation AttrLoc,
                                   VectorType::VectorKind VecKind) const;
 
   /// Return the unique reference to an extended vector type
   /// of the specified element type and size.
   ///
   /// \pre \p VectorType must be a built-in type.
   QualType getExtVectorType(QualType VectorType, unsigned NumElts) const;
 
   /// \pre Return a non-unique reference to the type for a dependently-sized
   /// vector of the specified element type.
   ///
   /// FIXME: We will need these to be uniqued, or at least comparable, at some
   /// point.
   QualType getDependentSizedExtVectorType(QualType VectorType,
                                           Expr *SizeExpr,
                                           SourceLocation AttrLoc) const;
 
   /// Return the unique reference to the matrix type of the specified element
   /// type and size
   ///
   /// \pre \p ElementType must be a valid matrix element type (see
   /// MatrixType::isValidElementType).
   QualType getConstantMatrixType(QualType ElementType, unsigned NumRows,
                                  unsigned NumColumns) const;
 
   /// Return the unique reference to the matrix type of the specified element
   /// type and size
   QualType getDependentSizedMatrixType(QualType ElementType, Expr *RowExpr,
                                        Expr *ColumnExpr,
                                        SourceLocation AttrLoc) const;
 
   QualType getDependentAddressSpaceType(QualType PointeeType,
                                         Expr *AddrSpaceExpr,
                                         SourceLocation AttrLoc) const;
 
   /// Return a K&R style C function type like 'int()'.
   QualType getFunctionNoProtoType(QualType ResultTy,
                                   const FunctionType::ExtInfo &Info) const;
 
   QualType getFunctionNoProtoType(QualType ResultTy) const {
     return getFunctionNoProtoType(ResultTy, FunctionType::ExtInfo());
   }
 
   /// Return a normal function type with a typed argument list.
   QualType getFunctionType(QualType ResultTy, ArrayRef<QualType> Args,
                            const FunctionProtoType::ExtProtoInfo &EPI) const {
     return getFunctionTypeInternal(ResultTy, Args, EPI, false);
   }
 
   QualType adjustStringLiteralBaseType(QualType StrLTy) const;
 
 private:
   /// Return a normal function type with a typed argument list.
   QualType getFunctionTypeInternal(QualType ResultTy, ArrayRef<QualType> Args,
                                    const FunctionProtoType::ExtProtoInfo &EPI,
                                    bool OnlyWantCanonical) const;
   QualType
   getAutoTypeInternal(QualType DeducedType, AutoTypeKeyword Keyword,
                       bool IsDependent, bool IsPack = false,
                       ConceptDecl *TypeConstraintConcept = nullptr,
                       ArrayRef<TemplateArgument> TypeConstraintArgs = {},
                       bool IsCanon = false) const;
 
 public:
   /// Return the unique reference to the type for the specified type
   /// declaration.
   QualType getTypeDeclType(const TypeDecl *Decl,
                            const TypeDecl *PrevDecl = nullptr) const {
     assert(Decl && "Passed null for Decl param");
     if (Decl->TypeForDecl) return QualType(Decl->TypeForDecl, 0);
 
     if (PrevDecl) {
       assert(PrevDecl->TypeForDecl && "previous decl has no TypeForDecl");
       Decl->TypeForDecl = PrevDecl->TypeForDecl;
       return QualType(PrevDecl->TypeForDecl, 0);
     }
 
     return getTypeDeclTypeSlow(Decl);
   }
 
   QualType getUsingType(const UsingShadowDecl *Found,
                         QualType Underlying) const;
 
   /// Return the unique reference to the type for the specified
   /// typedef-name decl.
   QualType getTypedefType(const TypedefNameDecl *Decl,
                           QualType Underlying = QualType()) const;
 
   QualType getRecordType(const RecordDecl *Decl) const;
 
   QualType getEnumType(const EnumDecl *Decl) const;
 
   QualType
   getUnresolvedUsingType(const UnresolvedUsingTypenameDecl *Decl) const;
 
   QualType getInjectedClassNameType(CXXRecordDecl *Decl, QualType TST) const;
 
   QualType getAttributedType(attr::Kind attrKind,
                              QualType modifiedType,
                              QualType equivalentType);
 
   QualType getSubstTemplateTypeParmType(const TemplateTypeParmType *Replaced,
                                         QualType Replacement) const;
   QualType getSubstTemplateTypeParmPackType(
                                           const TemplateTypeParmType *Replaced,
                                             const TemplateArgument &ArgPack);
 
   QualType
   getTemplateTypeParmType(unsigned Depth, unsigned Index,
                           bool ParameterPack,
                           TemplateTypeParmDecl *ParmDecl = nullptr) const;
 
   QualType getTemplateSpecializationType(TemplateName T,
                                          ArrayRef<TemplateArgument> Args,
                                          QualType Canon = QualType()) const;
 
   QualType
   getCanonicalTemplateSpecializationType(TemplateName T,
                                          ArrayRef<TemplateArgument> Args) const;
 
   QualType getTemplateSpecializationType(TemplateName T,
                                          const TemplateArgumentListInfo &Args,
                                          QualType Canon = QualType()) const;
 
   TypeSourceInfo *
   getTemplateSpecializationTypeInfo(TemplateName T, SourceLocation TLoc,
                                     const TemplateArgumentListInfo &Args,
                                     QualType Canon = QualType()) const;
 
   QualType getParenType(QualType NamedType) const;
 
   QualType getMacroQualifiedType(QualType UnderlyingTy,
                                  const IdentifierInfo *MacroII) const;
 
   QualType getElaboratedType(ElaboratedTypeKeyword Keyword,
                              NestedNameSpecifier *NNS, QualType NamedType,
                              TagDecl *OwnedTagDecl = nullptr) const;
   QualType getDependentNameType(ElaboratedTypeKeyword Keyword,
                                 NestedNameSpecifier *NNS,
                                 const IdentifierInfo *Name,
                                 QualType Canon = QualType()) const;
 
   QualType getDependentTemplateSpecializationType(ElaboratedTypeKeyword Keyword,
                                                   NestedNameSpecifier *NNS,
                                                   const IdentifierInfo *Name,
                                     const TemplateArgumentListInfo &Args) const;
   QualType getDependentTemplateSpecializationType(
       ElaboratedTypeKeyword Keyword, NestedNameSpecifier *NNS,
       const IdentifierInfo *Name, ArrayRef<TemplateArgument> Args) const;
 
   TemplateArgument getInjectedTemplateArg(NamedDecl *ParamDecl);
 
   /// Get a template argument list with one argument per template parameter
   /// in a template parameter list, such as for the injected class name of
   /// a class template.
   void getInjectedTemplateArgs(const TemplateParameterList *Params,
                                SmallVectorImpl<TemplateArgument> &Args);
 
   /// Form a pack expansion type with the given pattern.
   /// \param NumExpansions The number of expansions for the pack, if known.
   /// \param ExpectPackInType If \c false, we should not expect \p Pattern to
   ///        contain an unexpanded pack. This only makes sense if the pack
   ///        expansion is used in a context where the arity is inferred from
   ///        elsewhere, such as if the pattern contains a placeholder type or
   ///        if this is the canonical type of another pack expansion type.
   QualType getPackExpansionType(QualType Pattern,
                                 Optional<unsigned> NumExpansions,
                                 bool ExpectPackInType = true);
 
   QualType getObjCInterfaceType(const ObjCInterfaceDecl *Decl,
                                 ObjCInterfaceDecl *PrevDecl = nullptr) const;
 
   /// Legacy interface: cannot provide type arguments or __kindof.
   QualType getObjCObjectType(QualType Base,
                              ObjCProtocolDecl * const *Protocols,
                              unsigned NumProtocols) const;
 
   QualType getObjCObjectType(QualType Base,
                              ArrayRef<QualType> typeArgs,
                              ArrayRef<ObjCProtocolDecl *> protocols,
                              bool isKindOf) const;
 
   QualType getObjCTypeParamType(const ObjCTypeParamDecl *Decl,
                                 ArrayRef<ObjCProtocolDecl *> protocols) const;
   void adjustObjCTypeParamBoundType(const ObjCTypeParamDecl *Orig,
                                     ObjCTypeParamDecl *New) const;
 
   bool ObjCObjectAdoptsQTypeProtocols(QualType QT, ObjCInterfaceDecl *Decl);
 
   /// QIdProtocolsAdoptObjCObjectProtocols - Checks that protocols in
   /// QT's qualified-id protocol list adopt all protocols in IDecl's list
   /// of protocols.
   bool QIdProtocolsAdoptObjCObjectProtocols(QualType QT,
                                             ObjCInterfaceDecl *IDecl);
 
   /// Return a ObjCObjectPointerType type for the given ObjCObjectType.
   QualType getObjCObjectPointerType(QualType OIT) const;
 
   /// GCC extension.
   QualType getTypeOfExprType(Expr *e) const;
   QualType getTypeOfType(QualType t) const;
 
   QualType getReferenceQualifiedType(const Expr *e) const;
 
   /// C++11 decltype.
   QualType getDecltypeType(Expr *e, QualType UnderlyingType) const;
 
   /// Unary type transforms
   QualType getUnaryTransformType(QualType BaseType, QualType UnderlyingType,
                                  UnaryTransformType::UTTKind UKind) const;
 
   /// C++11 deduced auto type.
   QualType getAutoType(QualType DeducedType, AutoTypeKeyword Keyword,
                        bool IsDependent, bool IsPack = false,
                        ConceptDecl *TypeConstraintConcept = nullptr,
                        ArrayRef<TemplateArgument> TypeConstraintArgs ={}) const;
 
   /// C++11 deduction pattern for 'auto' type.
   QualType getAutoDeductType() const;
 
   /// C++11 deduction pattern for 'auto &&' type.
   QualType getAutoRRefDeductType() const;
 
   /// C++17 deduced class template specialization type.
   QualType getDeducedTemplateSpecializationType(TemplateName Template,
                                                 QualType DeducedType,
                                                 bool IsDependent) const;
 
   /// Return the unique reference to the type for the specified TagDecl
   /// (struct/union/class/enum) decl.
   QualType getTagDeclType(const TagDecl *Decl) const;
 
   /// Return the unique type for "size_t" (C99 7.17), defined in
   /// <stddef.h>.
   ///
   /// The sizeof operator requires this (C99 6.5.3.4p4).
   CanQualType getSizeType() const;
 
   /// Return the unique signed counterpart of
   /// the integer type corresponding to size_t.
   CanQualType getSignedSizeType() const;
 
   /// Return the unique type for "intmax_t" (C99 7.18.1.5), defined in
   /// <stdint.h>.
   CanQualType getIntMaxType() const;
 
   /// Return the unique type for "uintmax_t" (C99 7.18.1.5), defined in
   /// <stdint.h>.
   CanQualType getUIntMaxType() const;
 
   /// Return the unique wchar_t type available in C++ (and available as
   /// __wchar_t as a Microsoft extension).
   QualType getWCharType() const { return WCharTy; }
 
   /// Return the type of wide characters. In C++, this returns the
   /// unique wchar_t type. In C99, this returns a type compatible with the type
   /// defined in <stddef.h> as defined by the target.
   QualType getWideCharType() const { return WideCharTy; }
 
   /// Return the type of "signed wchar_t".
   ///
   /// Used when in C++, as a GCC extension.
   QualType getSignedWCharType() const;
 
   /// Return the type of "unsigned wchar_t".
   ///
   /// Used when in C++, as a GCC extension.
   QualType getUnsignedWCharType() const;
 
   /// In C99, this returns a type compatible with the type
   /// defined in <stddef.h> as defined by the target.
   QualType getWIntType() const { return WIntTy; }
 
   /// Return a type compatible with "intptr_t" (C99 7.18.1.4),
   /// as defined by the target.
   QualType getIntPtrType() const;
 
   /// Return a type compatible with "uintptr_t" (C99 7.18.1.4),
   /// as defined by the target.
   QualType getUIntPtrType() const;
 
   /// Return the unique type for "ptrdiff_t" (C99 7.17) defined in
   /// <stddef.h>. Pointer - pointer requires this (C99 6.5.6p9).
   QualType getPointerDiffType() const;
 
   /// Return the unique unsigned counterpart of "ptrdiff_t"
   /// integer type. The standard (C11 7.21.6.1p7) refers to this type
   /// in the definition of %tu format specifier.
   QualType getUnsignedPointerDiffType() const;
 
   /// Return the unique type for "pid_t" defined in
   /// <sys/types.h>. We need this to compute the correct type for vfork().
   QualType getProcessIDType() const;
 
   /// Return the C structure type used to represent constant CFStrings.
   QualType getCFConstantStringType() const;
 
   /// Returns the C struct type for objc_super
   QualType getObjCSuperType() const;
   void setObjCSuperType(QualType ST) { ObjCSuperType = ST; }
 
   /// Get the structure type used to representation CFStrings, or NULL
   /// if it hasn't yet been built.
   QualType getRawCFConstantStringType() const {
     if (CFConstantStringTypeDecl)
       return getTypedefType(CFConstantStringTypeDecl);
     return QualType();
   }
   void setCFConstantStringType(QualType T);
   TypedefDecl *getCFConstantStringDecl() const;
   RecordDecl *getCFConstantStringTagDecl() const;
 
   // This setter/getter represents the ObjC type for an NSConstantString.
   void setObjCConstantStringInterface(ObjCInterfaceDecl *Decl);
   QualType getObjCConstantStringInterface() const {
     return ObjCConstantStringType;
   }
 
   QualType getObjCNSStringType() const {
     return ObjCNSStringType;
   }
 
   void setObjCNSStringType(QualType T) {
     ObjCNSStringType = T;
   }
 
   /// Retrieve the type that \c id has been defined to, which may be
   /// different from the built-in \c id if \c id has been typedef'd.
   QualType getObjCIdRedefinitionType() const {
     if (ObjCIdRedefinitionType.isNull())
       return getObjCIdType();
     return ObjCIdRedefinitionType;
   }
 
   /// Set the user-written type that redefines \c id.
   void setObjCIdRedefinitionType(QualType RedefType) {
     ObjCIdRedefinitionType = RedefType;
   }
 
   /// Retrieve the type that \c Class has been defined to, which may be
   /// different from the built-in \c Class if \c Class has been typedef'd.
   QualType getObjCClassRedefinitionType() const {
     if (ObjCClassRedefinitionType.isNull())
       return getObjCClassType();
     return ObjCClassRedefinitionType;
   }
 
   /// Set the user-written type that redefines 'SEL'.
   void setObjCClassRedefinitionType(QualType RedefType) {
     ObjCClassRedefinitionType = RedefType;
   }
 
   /// Retrieve the type that 'SEL' has been defined to, which may be
   /// different from the built-in 'SEL' if 'SEL' has been typedef'd.
   QualType getObjCSelRedefinitionType() const {
     if (ObjCSelRedefinitionType.isNull())
       return getObjCSelType();
     return ObjCSelRedefinitionType;
   }
 
   /// Set the user-written type that redefines 'SEL'.
   void setObjCSelRedefinitionType(QualType RedefType) {
     ObjCSelRedefinitionType = RedefType;
   }
 
   /// Retrieve the identifier 'NSObject'.
   IdentifierInfo *getNSObjectName() const {
     if (!NSObjectName) {
       NSObjectName = &Idents.get("NSObject");
     }
 
     return NSObjectName;
   }
 
   /// Retrieve the identifier 'NSCopying'.
   IdentifierInfo *getNSCopyingName() {
     if (!NSCopyingName) {
       NSCopyingName = &Idents.get("NSCopying");
     }
 
     return NSCopyingName;
   }
 
   CanQualType getNSUIntegerType() const;
 
   CanQualType getNSIntegerType() const;
 
   /// Retrieve the identifier 'bool'.
   IdentifierInfo *getBoolName() const {
     if (!BoolName)
       BoolName = &Idents.get("bool");
     return BoolName;
   }
 
   IdentifierInfo *getMakeIntegerSeqName() const {
     if (!MakeIntegerSeqName)
       MakeIntegerSeqName = &Idents.get("__make_integer_seq");
     return MakeIntegerSeqName;
   }
 
   IdentifierInfo *getTypePackElementName() const {
     if (!TypePackElementName)
       TypePackElementName = &Idents.get("__type_pack_element");
     return TypePackElementName;
   }
 
   /// Retrieve the Objective-C "instancetype" type, if already known;
   /// otherwise, returns a NULL type;
   QualType getObjCInstanceType() {
     return getTypeDeclType(getObjCInstanceTypeDecl());
   }
 
   /// Retrieve the typedef declaration corresponding to the Objective-C
   /// "instancetype" type.
   TypedefDecl *getObjCInstanceTypeDecl();
 
   /// Set the type for the C FILE type.
   void setFILEDecl(TypeDecl *FILEDecl) { this->FILEDecl = FILEDecl; }
 
   /// Retrieve the C FILE type.
   QualType getFILEType() const {
     if (FILEDecl)
       return getTypeDeclType(FILEDecl);
     return QualType();
   }
 
   /// Set the type for the C jmp_buf type.
   void setjmp_bufDecl(TypeDecl *jmp_bufDecl) {
     this->jmp_bufDecl = jmp_bufDecl;
   }
 
   /// Retrieve the C jmp_buf type.
   QualType getjmp_bufType() const {
     if (jmp_bufDecl)
       return getTypeDeclType(jmp_bufDecl);
     return QualType();
   }
 
   /// Set the type for the C sigjmp_buf type.
   void setsigjmp_bufDecl(TypeDecl *sigjmp_bufDecl) {
     this->sigjmp_bufDecl = sigjmp_bufDecl;
   }
 
   /// Retrieve the C sigjmp_buf type.
   QualType getsigjmp_bufType() const {
     if (sigjmp_bufDecl)
       return getTypeDeclType(sigjmp_bufDecl);
     return QualType();
   }
 
   /// Set the type for the C ucontext_t type.
   void setucontext_tDecl(TypeDecl *ucontext_tDecl) {
     this->ucontext_tDecl = ucontext_tDecl;
   }
 
   /// Retrieve the C ucontext_t type.
   QualType getucontext_tType() const {
     if (ucontext_tDecl)
       return getTypeDeclType(ucontext_tDecl);
     return QualType();
   }
 
   /// The result type of logical operations, '<', '>', '!=', etc.
   QualType getLogicalOperationType() const {
     return getLangOpts().CPlusPlus ? BoolTy : IntTy;
   }
 
   /// Emit the Objective-CC type encoding for the given type \p T into
   /// \p S.
   ///
   /// If \p Field is specified then record field names are also encoded.
   void getObjCEncodingForType(QualType T, std::string &S,
                               const FieldDecl *Field=nullptr,
                               QualType *NotEncodedT=nullptr) const;
 
   /// Emit the Objective-C property type encoding for the given
   /// type \p T into \p S.
   void getObjCEncodingForPropertyType(QualType T, std::string &S) const;
 
   void getLegacyIntegralTypeEncoding(QualType &t) const;
 
   /// Put the string version of the type qualifiers \p QT into \p S.
   void getObjCEncodingForTypeQualifier(Decl::ObjCDeclQualifier QT,
                                        std::string &S) const;
 
   /// Emit the encoded type for the function \p Decl into \p S.
   ///
   /// This is in the same format as Objective-C method encodings.
   ///
   /// \returns true if an error occurred (e.g., because one of the parameter
   /// types is incomplete), false otherwise.
   std::string getObjCEncodingForFunctionDecl(const FunctionDecl *Decl) const;
 
   /// Emit the encoded type for the method declaration \p Decl into
   /// \p S.
   std::string getObjCEncodingForMethodDecl(const ObjCMethodDecl *Decl,
                                            bool Extended = false) const;
 
   /// Return the encoded type for this block declaration.
   std::string getObjCEncodingForBlock(const BlockExpr *blockExpr) const;
 
   /// getObjCEncodingForPropertyDecl - Return the encoded type for
   /// this method declaration. If non-NULL, Container must be either
   /// an ObjCCategoryImplDecl or ObjCImplementationDecl; it should
   /// only be NULL when getting encodings for protocol properties.
   std::string getObjCEncodingForPropertyDecl(const ObjCPropertyDecl *PD,
                                              const Decl *Container) const;
 
   bool ProtocolCompatibleWithProtocol(ObjCProtocolDecl *lProto,
                                       ObjCProtocolDecl *rProto) const;
 
   ObjCPropertyImplDecl *getObjCPropertyImplDeclForPropertyDecl(
                                                   const ObjCPropertyDecl *PD,
                                                   const Decl *Container) const;
 
   /// Return the size of type \p T for Objective-C encoding purpose,
   /// in characters.
   CharUnits getObjCEncodingTypeSize(QualType T) const;
 
   /// Retrieve the typedef corresponding to the predefined \c id type
   /// in Objective-C.
   TypedefDecl *getObjCIdDecl() const;
 
   /// Represents the Objective-CC \c id type.
   ///
   /// This is set up lazily, by Sema.  \c id is always a (typedef for a)
   /// pointer type, a pointer to a struct.
   QualType getObjCIdType() const {
     return getTypeDeclType(getObjCIdDecl());
   }
 
   /// Retrieve the typedef corresponding to the predefined 'SEL' type
   /// in Objective-C.
   TypedefDecl *getObjCSelDecl() const;
 
   /// Retrieve the type that corresponds to the predefined Objective-C
   /// 'SEL' type.
   QualType getObjCSelType() const {
     return getTypeDeclType(getObjCSelDecl());
   }
 
   /// Retrieve the typedef declaration corresponding to the predefined
   /// Objective-C 'Class' type.
   TypedefDecl *getObjCClassDecl() const;
 
   /// Represents the Objective-C \c Class type.
   ///
   /// This is set up lazily, by Sema.  \c Class is always a (typedef for a)
   /// pointer type, a pointer to a struct.
   QualType getObjCClassType() const {
     return getTypeDeclType(getObjCClassDecl());
   }
 
   /// Retrieve the Objective-C class declaration corresponding to
   /// the predefined \c Protocol class.
   ObjCInterfaceDecl *getObjCProtocolDecl() const;
 
   /// Retrieve declaration of 'BOOL' typedef
   TypedefDecl *getBOOLDecl() const {
     return BOOLDecl;
   }
 
   /// Save declaration of 'BOOL' typedef
   void setBOOLDecl(TypedefDecl *TD) {
     BOOLDecl = TD;
   }
 
   /// type of 'BOOL' type.
   QualType getBOOLType() const {
     return getTypeDeclType(getBOOLDecl());
   }
 
   /// Retrieve the type of the Objective-C \c Protocol class.
   QualType getObjCProtoType() const {
     return getObjCInterfaceType(getObjCProtocolDecl());
   }
 
   /// Retrieve the C type declaration corresponding to the predefined
   /// \c __builtin_va_list type.
   TypedefDecl *getBuiltinVaListDecl() const;
 
   /// Retrieve the type of the \c __builtin_va_list type.
   QualType getBuiltinVaListType() const {
     return getTypeDeclType(getBuiltinVaListDecl());
   }
 
   /// Retrieve the C type declaration corresponding to the predefined
   /// \c __va_list_tag type used to help define the \c __builtin_va_list type
   /// for some targets.
   Decl *getVaListTagDecl() const;
 
   /// Retrieve the C type declaration corresponding to the predefined
   /// \c __builtin_ms_va_list type.
   TypedefDecl *getBuiltinMSVaListDecl() const;
 
   /// Retrieve the type of the \c __builtin_ms_va_list type.
   QualType getBuiltinMSVaListType() const {
     return getTypeDeclType(getBuiltinMSVaListDecl());
   }
 
   /// Retrieve the implicitly-predeclared 'struct _GUID' declaration.
   TagDecl *getMSGuidTagDecl() const { return MSGuidTagDecl; }
 
   /// Retrieve the implicitly-predeclared 'struct _GUID' type.
   QualType getMSGuidType() const {
     assert(MSGuidTagDecl && "asked for GUID type but MS extensions disabled");
     return getTagDeclType(MSGuidTagDecl);
   }
 
   /// Return whether a declaration to a builtin is allowed to be
   /// overloaded/redeclared.
   bool canBuiltinBeRedeclared(const FunctionDecl *) const;
 
   /// Return a type with additional \c const, \c volatile, or
   /// \c restrict qualifiers.
   QualType getCVRQualifiedType(QualType T, unsigned CVR) const {
     return getQualifiedType(T, Qualifiers::fromCVRMask(CVR));
   }
 
   /// Un-split a SplitQualType.
   QualType getQualifiedType(SplitQualType split) const {
     return getQualifiedType(split.Ty, split.Quals);
   }
 
   /// Return a type with additional qualifiers.
   QualType getQualifiedType(QualType T, Qualifiers Qs) const {
     if (!Qs.hasNonFastQualifiers())
       return T.withFastQualifiers(Qs.getFastQualifiers());
     QualifierCollector Qc(Qs);
     const Type *Ptr = Qc.strip(T);
     return getExtQualType(Ptr, Qc);
   }
 
   /// Return a type with additional qualifiers.
   QualType getQualifiedType(const Type *T, Qualifiers Qs) const {
     if (!Qs.hasNonFastQualifiers())
       return QualType(T, Qs.getFastQualifiers());
     return getExtQualType(T, Qs);
   }
 
   /// Return a type with the given lifetime qualifier.
   ///
   /// \pre Neither type.ObjCLifetime() nor \p lifetime may be \c OCL_None.
   QualType getLifetimeQualifiedType(QualType type,
                                     Qualifiers::ObjCLifetime lifetime) {
     assert(type.getObjCLifetime() == Qualifiers::OCL_None);
     assert(lifetime != Qualifiers::OCL_None);
 
     Qualifiers qs;
     qs.addObjCLifetime(lifetime);
     return getQualifiedType(type, qs);
   }
 
   /// getUnqualifiedObjCPointerType - Returns version of
   /// Objective-C pointer type with lifetime qualifier removed.
   QualType getUnqualifiedObjCPointerType(QualType type) const {
     if (!type.getTypePtr()->isObjCObjectPointerType() ||
         !type.getQualifiers().hasObjCLifetime())
       return type;
     Qualifiers Qs = type.getQualifiers();
     Qs.removeObjCLifetime();
     return getQualifiedType(type.getUnqualifiedType(), Qs);
   }
 
   unsigned char getFixedPointScale(QualType Ty) const;
   unsigned char getFixedPointIBits(QualType Ty) const;
   llvm::FixedPointSemantics getFixedPointSemantics(QualType Ty) const;
   llvm::APFixedPoint getFixedPointMax(QualType Ty) const;
   llvm::APFixedPoint getFixedPointMin(QualType Ty) const;
 
   DeclarationNameInfo getNameForTemplate(TemplateName Name,
                                          SourceLocation NameLoc) const;
 
   TemplateName getOverloadedTemplateName(UnresolvedSetIterator Begin,
                                          UnresolvedSetIterator End) const;
   TemplateName getAssumedTemplateName(DeclarationName Name) const;
 
   TemplateName getQualifiedTemplateName(NestedNameSpecifier *NNS,
                                         bool TemplateKeyword,
                                         TemplateDecl *Template) const;
 
   TemplateName getDependentTemplateName(NestedNameSpecifier *NNS,
                                         const IdentifierInfo *Name) const;
   TemplateName getDependentTemplateName(NestedNameSpecifier *NNS,
                                         OverloadedOperatorKind Operator) const;
   TemplateName getSubstTemplateTemplateParm(TemplateTemplateParmDecl *param,
                                             TemplateName replacement) const;
   TemplateName getSubstTemplateTemplateParmPack(TemplateTemplateParmDecl *Param,
                                         const TemplateArgument &ArgPack) const;
 
   enum GetBuiltinTypeError {
     /// No error
     GE_None,
 
     /// Missing a type
     GE_Missing_type,
 
     /// Missing a type from <stdio.h>
     GE_Missing_stdio,
 
     /// Missing a type from <setjmp.h>
     GE_Missing_setjmp,
 
     /// Missing a type from <ucontext.h>
     GE_Missing_ucontext
   };
 
   QualType DecodeTypeStr(const char *&Str, const ASTContext &Context,
                          ASTContext::GetBuiltinTypeError &Error,
                          bool &RequireICE, bool AllowTypeModifiers) const;
 
   /// Return the type for the specified builtin.
   ///
   /// If \p IntegerConstantArgs is non-null, it is filled in with a bitmask of
   /// arguments to the builtin that are required to be integer constant
   /// expressions.
   QualType GetBuiltinType(unsigned ID, GetBuiltinTypeError &Error,
                           unsigned *IntegerConstantArgs = nullptr) const;
 
   /// Types and expressions required to build C++2a three-way comparisons
   /// using operator<=>, including the values return by builtin <=> operators.
   ComparisonCategories CompCategories;
 
 private:
   CanQualType getFromTargetType(unsigned Type) const;
   TypeInfo getTypeInfoImpl(const Type *T) const;
 
   //===--------------------------------------------------------------------===//
   //                         Type Predicates.
   //===--------------------------------------------------------------------===//
 
 public:
   /// Return one of the GCNone, Weak or Strong Objective-C garbage
   /// collection attributes.
   Qualifiers::GC getObjCGCAttrKind(QualType Ty) const;
 
   /// Return true if the given vector types are of the same unqualified
   /// type or if they are equivalent to the same GCC vector type.
   ///
   /// \note This ignores whether they are target-specific (AltiVec or Neon)
   /// types.
   bool areCompatibleVectorTypes(QualType FirstVec, QualType SecondVec);
 
   /// Return true if the given types are an SVE builtin and a VectorType that
   /// is a fixed-length representation of the SVE builtin for a specific
   /// vector-length.
   bool areCompatibleSveTypes(QualType FirstType, QualType SecondType);
 
   /// Return true if the given vector types are lax-compatible SVE vector types,
   /// false otherwise.
   bool areLaxCompatibleSveTypes(QualType FirstType, QualType SecondType);
 
   /// Return true if the type has been explicitly qualified with ObjC ownership.
   /// A type may be implicitly qualified with ownership under ObjC ARC, and in
   /// some cases the compiler treats these differently.
   bool hasDirectOwnershipQualifier(QualType Ty) const;
 
   /// Return true if this is an \c NSObject object with its \c NSObject
   /// attribute set.
   static bool isObjCNSObjectType(QualType Ty) {
     return Ty->isObjCNSObjectType();
   }
 
   //===--------------------------------------------------------------------===//
   //                         Type Sizing and Analysis
   //===--------------------------------------------------------------------===//
 
   /// Return the APFloat 'semantics' for the specified scalar floating
   /// point type.
   const llvm::fltSemantics &getFloatTypeSemantics(QualType T) const;
 
   /// Get the size and alignment of the specified complete type in bits.
   TypeInfo getTypeInfo(const Type *T) const;
   TypeInfo getTypeInfo(QualType T) const { return getTypeInfo(T.getTypePtr()); }
 
   /// Get default simd alignment of the specified complete type in bits.
   unsigned getOpenMPDefaultSimdAlign(QualType T) const;
 
   /// Return the size of the specified (complete) type \p T, in bits.
   uint64_t getTypeSize(QualType T) const { return getTypeInfo(T).Width; }
   uint64_t getTypeSize(const Type *T) const { return getTypeInfo(T).Width; }
 
   /// Return the size of the character type, in bits.
   uint64_t getCharWidth() const {
     return getTypeSize(CharTy);
   }
 
   /// Convert a size in bits to a size in characters.
   CharUnits toCharUnitsFromBits(int64_t BitSize) const;
 
   /// Convert a size in characters to a size in bits.
   int64_t toBits(CharUnits CharSize) const;
 
   /// Return the size of the specified (complete) type \p T, in
   /// characters.
   CharUnits getTypeSizeInChars(QualType T) const;
   CharUnits getTypeSizeInChars(const Type *T) const;
 
   Optional<CharUnits> getTypeSizeInCharsIfKnown(QualType Ty) const {
     if (Ty->isIncompleteType() || Ty->isDependentType())
       return None;
     return getTypeSizeInChars(Ty);
   }
 
   Optional<CharUnits> getTypeSizeInCharsIfKnown(const Type *Ty) const {
     return getTypeSizeInCharsIfKnown(QualType(Ty, 0));
   }
 
   /// Return the ABI-specified alignment of a (complete) type \p T, in
   /// bits.
   unsigned getTypeAlign(QualType T) const { return getTypeInfo(T).Align; }
   unsigned getTypeAlign(const Type *T) const { return getTypeInfo(T).Align; }
 
   /// Return the ABI-specified natural alignment of a (complete) type \p T,
   /// before alignment adjustments, in bits.
   ///
   /// This alignment is curently used only by ARM and AArch64 when passing
   /// arguments of a composite type.
   unsigned getTypeUnadjustedAlign(QualType T) const {
     return getTypeUnadjustedAlign(T.getTypePtr());
   }
   unsigned getTypeUnadjustedAlign(const Type *T) const;
 
   /// Return the alignment of a type, in bits, or 0 if
   /// the type is incomplete and we cannot determine the alignment (for
   /// example, from alignment attributes). The returned alignment is the
   /// Preferred alignment if NeedsPreferredAlignment is true, otherwise is the
   /// ABI alignment.
   unsigned getTypeAlignIfKnown(QualType T,
                                bool NeedsPreferredAlignment = false) const;
 
   /// Return the ABI-specified alignment of a (complete) type \p T, in
   /// characters.
   CharUnits getTypeAlignInChars(QualType T) const;
   CharUnits getTypeAlignInChars(const Type *T) const;
 
   /// Return the PreferredAlignment of a (complete) type \p T, in
   /// characters.
   CharUnits getPreferredTypeAlignInChars(QualType T) const {
     return toCharUnitsFromBits(getPreferredTypeAlign(T));
   }
 
   /// getTypeUnadjustedAlignInChars - Return the ABI-specified alignment of a type,
   /// in characters, before alignment adjustments. This method does not work on
   /// incomplete types.
   CharUnits getTypeUnadjustedAlignInChars(QualType T) const;
   CharUnits getTypeUnadjustedAlignInChars(const Type *T) const;
 
   // getTypeInfoDataSizeInChars - Return the size of a type, in chars. If the
   // type is a record, its data size is returned.
   TypeInfoChars getTypeInfoDataSizeInChars(QualType T) const;
 
   TypeInfoChars getTypeInfoInChars(const Type *T) const;
   TypeInfoChars getTypeInfoInChars(QualType T) const;
 
   /// Determine if the alignment the type has was required using an
   /// alignment attribute.
   bool isAlignmentRequired(const Type *T) const;
   bool isAlignmentRequired(QualType T) const;
 
   /// Return the "preferred" alignment of the specified type \p T for
   /// the current target, in bits.
   ///
   /// This can be different than the ABI alignment in cases where it is
   /// beneficial for performance or backwards compatibility preserving to
   /// overalign a data type. (Note: despite the name, the preferred alignment
   /// is ABI-impacting, and not an optimization.)
   unsigned getPreferredTypeAlign(QualType T) const {
     return getPreferredTypeAlign(T.getTypePtr());
   }
   unsigned getPreferredTypeAlign(const Type *T) const;
 
   /// Return the default alignment for __attribute__((aligned)) on
   /// this target, to be used if no alignment value is specified.
   unsigned getTargetDefaultAlignForAttributeAligned() const;
 
   /// Return the alignment in bits that should be given to a
   /// global variable with type \p T.
   unsigned getAlignOfGlobalVar(QualType T) const;
 
   /// Return the alignment in characters that should be given to a
   /// global variable with type \p T.
   CharUnits getAlignOfGlobalVarInChars(QualType T) const;
 
   /// Return a conservative estimate of the alignment of the specified
   /// decl \p D.
   ///
   /// \pre \p D must not be a bitfield type, as bitfields do not have a valid
   /// alignment.
   ///
   /// If \p ForAlignof, references are treated like their underlying type
   /// and  large arrays don't get any special treatment. If not \p ForAlignof
   /// it computes the value expected by CodeGen: references are treated like
   /// pointers and large arrays get extra alignment.
   CharUnits getDeclAlign(const Decl *D, bool ForAlignof = false) const;
 
   /// Return the alignment (in bytes) of the thrown exception object. This is
   /// only meaningful for targets that allocate C++ exceptions in a system
   /// runtime, such as those using the Itanium C++ ABI.
   CharUnits getExnObjectAlignment() const;
 
   /// Get or compute information about the layout of the specified
   /// record (struct/union/class) \p D, which indicates its size and field
   /// position information.
   const ASTRecordLayout &getASTRecordLayout(const RecordDecl *D) const;
 
   /// Get or compute information about the layout of the specified
   /// Objective-C interface.
   const ASTRecordLayout &getASTObjCInterfaceLayout(const ObjCInterfaceDecl *D)
     const;
 
   void DumpRecordLayout(const RecordDecl *RD, raw_ostream &OS,
                         bool Simple = false) const;
 
   /// Get or compute information about the layout of the specified
   /// Objective-C implementation.
   ///
   /// This may differ from the interface if synthesized ivars are present.
   const ASTRecordLayout &
   getASTObjCImplementationLayout(const ObjCImplementationDecl *D) const;
 
   /// Get our current best idea for the key function of the
   /// given record decl, or nullptr if there isn't one.
   ///
   /// The key function is, according to the Itanium C++ ABI section 5.2.3:
   ///   ...the first non-pure virtual function that is not inline at the
   ///   point of class definition.
   ///
   /// Other ABIs use the same idea.  However, the ARM C++ ABI ignores
   /// virtual functions that are defined 'inline', which means that
   /// the result of this computation can change.
   const CXXMethodDecl *getCurrentKeyFunction(const CXXRecordDecl *RD);
 
   /// Observe that the given method cannot be a key function.
   /// Checks the key-function cache for the method's class and clears it
   /// if matches the given declaration.
   ///
   /// This is used in ABIs where out-of-line definitions marked
   /// inline are not considered to be key functions.
   ///
   /// \param method should be the declaration from the class definition
   void setNonKeyFunction(const CXXMethodDecl *method);
 
   /// Loading virtual member pointers using the virtual inheritance model
   /// always results in an adjustment using the vbtable even if the index is
   /// zero.
   ///
   /// This is usually OK because the first slot in the vbtable points
   /// backwards to the top of the MDC.  However, the MDC might be reusing a
   /// vbptr from an nv-base.  In this case, the first slot in the vbtable
   /// points to the start of the nv-base which introduced the vbptr and *not*
   /// the MDC.  Modify the NonVirtualBaseAdjustment to account for this.
   CharUnits getOffsetOfBaseWithVBPtr(const CXXRecordDecl *RD) const;
 
   /// Get the offset of a FieldDecl or IndirectFieldDecl, in bits.
   uint64_t getFieldOffset(const ValueDecl *FD) const;
 
   /// Get the offset of an ObjCIvarDecl in bits.
   uint64_t lookupFieldBitOffset(const ObjCInterfaceDecl *OID,
                                 const ObjCImplementationDecl *ID,
                                 const ObjCIvarDecl *Ivar) const;
 
   /// Find the 'this' offset for the member path in a pointer-to-member
   /// APValue.
   CharUnits getMemberPointerPathAdjustment(const APValue &MP) const;
 
   bool isNearlyEmpty(const CXXRecordDecl *RD) const;
 
   VTableContextBase *getVTableContext();
 
   /// If \p T is null pointer, assume the target in ASTContext.
   MangleContext *createMangleContext(const TargetInfo *T = nullptr);
 
   /// Creates a device mangle context to correctly mangle lambdas in a mixed
   /// architecture compile by setting the lambda mangling number source to the
   /// DeviceLambdaManglingNumber. Currently this asserts that the TargetInfo
   /// (from the AuxTargetInfo) is a an itanium target.
   MangleContext *createDeviceMangleContext(const TargetInfo &T);
 
   void DeepCollectObjCIvars(const ObjCInterfaceDecl *OI, bool leafClass,
                             SmallVectorImpl<const ObjCIvarDecl*> &Ivars) const;
 
   unsigned CountNonClassIvars(const ObjCInterfaceDecl *OI) const;
   void CollectInheritedProtocols(const Decl *CDecl,
                           llvm::SmallPtrSet<ObjCProtocolDecl*, 8> &Protocols);
 
   /// Return true if the specified type has unique object representations
   /// according to (C++17 [meta.unary.prop]p9)
   bool hasUniqueObjectRepresentations(QualType Ty) const;
 
   //===--------------------------------------------------------------------===//
   //                            Type Operators
   //===--------------------------------------------------------------------===//
 
   /// Return the canonical (structural) type corresponding to the
   /// specified potentially non-canonical type \p T.
   ///
   /// The non-canonical version of a type may have many "decorated" versions of
   /// types.  Decorators can include typedefs, 'typeof' operators, etc. The
   /// returned type is guaranteed to be free of any of these, allowing two
   /// canonical types to be compared for exact equality with a simple pointer
   /// comparison.
   CanQualType getCanonicalType(QualType T) const {
     return CanQualType::CreateUnsafe(T.getCanonicalType());
   }
 
   const Type *getCanonicalType(const Type *T) const {
     return T->getCanonicalTypeInternal().getTypePtr();
   }
 
   /// Return the canonical parameter type corresponding to the specific
   /// potentially non-canonical one.
   ///
   /// Qualifiers are stripped off, functions are turned into function
   /// pointers, and arrays decay one level into pointers.
   CanQualType getCanonicalParamType(QualType T) const;
 
   /// Determine whether the given types \p T1 and \p T2 are equivalent.
   bool hasSameType(QualType T1, QualType T2) const {
     return getCanonicalType(T1) == getCanonicalType(T2);
   }
   bool hasSameType(const Type *T1, const Type *T2) const {
     return getCanonicalType(T1) == getCanonicalType(T2);
   }
 
   /// Return this type as a completely-unqualified array type,
   /// capturing the qualifiers in \p Quals.
   ///
   /// This will remove the minimal amount of sugaring from the types, similar
   /// to the behavior of QualType::getUnqualifiedType().
   ///
   /// \param T is the qualified type, which may be an ArrayType
   ///
   /// \param Quals will receive the full set of qualifiers that were
   /// applied to the array.
   ///
   /// \returns if this is an array type, the completely unqualified array type
   /// that corresponds to it. Otherwise, returns T.getUnqualifiedType().
   QualType getUnqualifiedArrayType(QualType T, Qualifiers &Quals);
 
   /// Determine whether the given types are equivalent after
   /// cvr-qualifiers have been removed.
   bool hasSameUnqualifiedType(QualType T1, QualType T2) const {
     return getCanonicalType(T1).getTypePtr() ==
            getCanonicalType(T2).getTypePtr();
   }
 
   bool hasSameNullabilityTypeQualifier(QualType SubT, QualType SuperT,
                                        bool IsParam) const {
     auto SubTnullability = SubT->getNullability(*this);
     auto SuperTnullability = SuperT->getNullability(*this);
     if (SubTnullability.hasValue() == SuperTnullability.hasValue()) {
       // Neither has nullability; return true
       if (!SubTnullability)
         return true;
       // Both have nullability qualifier.
       if (*SubTnullability == *SuperTnullability ||
           *SubTnullability == NullabilityKind::Unspecified ||
           *SuperTnullability == NullabilityKind::Unspecified)
         return true;
 
       if (IsParam) {
         // Ok for the superclass method parameter to be "nonnull" and the subclass
         // method parameter to be "nullable"
         return (*SuperTnullability == NullabilityKind::NonNull &&
                 *SubTnullability == NullabilityKind::Nullable);
       }
       // For the return type, it's okay for the superclass method to specify
       // "nullable" and the subclass method specify "nonnull"
       return (*SuperTnullability == NullabilityKind::Nullable &&
               *SubTnullability == NullabilityKind::NonNull);
     }
     return true;
   }
 
   bool ObjCMethodsAreEqual(const ObjCMethodDecl *MethodDecl,
                            const ObjCMethodDecl *MethodImp);
 
   bool UnwrapSimilarTypes(QualType &T1, QualType &T2,
                           bool AllowPiMismatch = true);
   void UnwrapSimilarArrayTypes(QualType &T1, QualType &T2,
                                bool AllowPiMismatch = true);
 
   /// Determine if two types are similar, according to the C++ rules. That is,
   /// determine if they are the same other than qualifiers on the initial
   /// sequence of pointer / pointer-to-member / array (and in Clang, object
   /// pointer) types and their element types.
   ///
   /// Clang offers a number of qualifiers in addition to the C++ qualifiers;
   /// those qualifiers are also ignored in the 'similarity' check.
   bool hasSimilarType(QualType T1, QualType T2);
 
   /// Determine if two types are similar, ignoring only CVR qualifiers.
   bool hasCvrSimilarType(QualType T1, QualType T2);
 
   /// Retrieves the "canonical" nested name specifier for a
   /// given nested name specifier.
   ///
   /// The canonical nested name specifier is a nested name specifier
   /// that uniquely identifies a type or namespace within the type
   /// system. For example, given:
   ///
   /// \code
   /// namespace N {
   ///   struct S {
   ///     template<typename T> struct X { typename T* type; };
   ///   };
   /// }
   ///
   /// template<typename T> struct Y {
   ///   typename N::S::X<T>::type member;
   /// };
   /// \endcode
   ///
   /// Here, the nested-name-specifier for N::S::X<T>:: will be
   /// S::X<template-param-0-0>, since 'S' and 'X' are uniquely defined
   /// by declarations in the type system and the canonical type for
   /// the template type parameter 'T' is template-param-0-0.
   NestedNameSpecifier *
   getCanonicalNestedNameSpecifier(NestedNameSpecifier *NNS) const;
 
   /// Retrieves the default calling convention for the current target.
   CallingConv getDefaultCallingConvention(bool IsVariadic,
                                           bool IsCXXMethod,
                                           bool IsBuiltin = false) const;
 
   /// Retrieves the "canonical" template name that refers to a
   /// given template.
   ///
   /// The canonical template name is the simplest expression that can
   /// be used to refer to a given template. For most templates, this
   /// expression is just the template declaration itself. For example,
   /// the template std::vector can be referred to via a variety of
   /// names---std::vector, \::std::vector, vector (if vector is in
   /// scope), etc.---but all of these names map down to the same
   /// TemplateDecl, which is used to form the canonical template name.
   ///
   /// Dependent template names are more interesting. Here, the
   /// template name could be something like T::template apply or
   /// std::allocator<T>::template rebind, where the nested name
   /// specifier itself is dependent. In this case, the canonical
   /// template name uses the shortest form of the dependent
   /// nested-name-specifier, which itself contains all canonical
   /// types, values, and templates.
   TemplateName getCanonicalTemplateName(const TemplateName &Name) const;
 
   /// Determine whether the given template names refer to the same
   /// template.
   bool hasSameTemplateName(const TemplateName &X, const TemplateName &Y) const;
 
   /// Determine whether the two declarations refer to the same entity.
   ///
   /// FIXME: isSameEntity is not const due to its implementation calls
   /// hasSameFunctionTypeIgnoringExceptionSpec which may alter this.
   bool isSameEntity(const NamedDecl *X, const NamedDecl *Y);
 
   /// Determine whether two template parameter lists are similar enough
   /// that they may be used in declarations of the same template.
   ///
   /// FIXME: isSameTemplateParameterList is not const since it calls
   /// isSameTemplateParameter.
   bool isSameTemplateParameterList(const TemplateParameterList *X,
                                    const TemplateParameterList *Y);
 
   /// Determine whether two template parameters are similar enough
   /// that they may be used in declarations of the same template.
   ///
   /// FIXME: isSameTemplateParameterList is not const since it calls
   /// isSameEntity.
   bool isSameTemplateParameter(const NamedDecl *X, const NamedDecl *Y);
 
   /// Retrieve the "canonical" template argument.
   ///
   /// The canonical template argument is the simplest template argument
   /// (which may be a type, value, expression, or declaration) that
   /// expresses the value of the argument.
   TemplateArgument getCanonicalTemplateArgument(const TemplateArgument &Arg)
     const;
 
   /// Type Query functions.  If the type is an instance of the specified class,
   /// return the Type pointer for the underlying maximally pretty type.  This
   /// is a member of ASTContext because this may need to do some amount of
   /// canonicalization, e.g. to move type qualifiers into the element type.
   const ArrayType *getAsArrayType(QualType T) const;
   const ConstantArrayType *getAsConstantArrayType(QualType T) const {
     return dyn_cast_or_null<ConstantArrayType>(getAsArrayType(T));
   }
   const VariableArrayType *getAsVariableArrayType(QualType T) const {
     return dyn_cast_or_null<VariableArrayType>(getAsArrayType(T));
   }
   const IncompleteArrayType *getAsIncompleteArrayType(QualType T) const {
     return dyn_cast_or_null<IncompleteArrayType>(getAsArrayType(T));
   }
   const DependentSizedArrayType *getAsDependentSizedArrayType(QualType T)
     const {
     return dyn_cast_or_null<DependentSizedArrayType>(getAsArrayType(T));
   }
 
   /// Return the innermost element type of an array type.
   ///
   /// For example, will return "int" for int[m][n]
   QualType getBaseElementType(const ArrayType *VAT) const;
 
   /// Return the innermost element type of a type (which needn't
   /// actually be an array type).
   QualType getBaseElementType(QualType QT) const;
 
   /// Return number of constant array elements.
   uint64_t getConstantArrayElementCount(const ConstantArrayType *CA) const;
 
   /// Perform adjustment on the parameter type of a function.
   ///
   /// This routine adjusts the given parameter type @p T to the actual
   /// parameter type used by semantic analysis (C99 6.7.5.3p[7,8],
   /// C++ [dcl.fct]p3). The adjusted parameter type is returned.
   QualType getAdjustedParameterType(QualType T) const;
 
   /// Retrieve the parameter type as adjusted for use in the signature
   /// of a function, decaying array and function types and removing top-level
   /// cv-qualifiers.
   QualType getSignatureParameterType(QualType T) const;
 
   QualType getExceptionObjectType(QualType T) const;
 
   /// Return the properly qualified result of decaying the specified
   /// array type to a pointer.
   ///
   /// This operation is non-trivial when handling typedefs etc.  The canonical
   /// type of \p T must be an array type, this returns a pointer to a properly
   /// qualified element of the array.
   ///
   /// See C99 6.7.5.3p7 and C99 6.3.2.1p3.
   QualType getArrayDecayedType(QualType T) const;
 
   /// Return the type that \p PromotableType will promote to: C99
   /// 6.3.1.1p2, assuming that \p PromotableType is a promotable integer type.
   QualType getPromotedIntegerType(QualType PromotableType) const;
 
   /// Recurses in pointer/array types until it finds an Objective-C
   /// retainable type and returns its ownership.
   Qualifiers::ObjCLifetime getInnerObjCOwnership(QualType T) const;
 
   /// Whether this is a promotable bitfield reference according
   /// to C99 6.3.1.1p2, bullet 2 (and GCC extensions).
   ///
   /// \returns the type this bit-field will promote to, or NULL if no
   /// promotion occurs.
   QualType isPromotableBitField(Expr *E) const;
 
   /// Return the highest ranked integer type, see C99 6.3.1.8p1.
   ///
   /// If \p LHS > \p RHS, returns 1.  If \p LHS == \p RHS, returns 0.  If
   /// \p LHS < \p RHS, return -1.
   int getIntegerTypeOrder(QualType LHS, QualType RHS) const;
 
   /// Compare the rank of the two specified floating point types,
   /// ignoring the domain of the type (i.e. 'double' == '_Complex double').
   ///
   /// If \p LHS > \p RHS, returns 1.  If \p LHS == \p RHS, returns 0.  If
   /// \p LHS < \p RHS, return -1.
   int getFloatingTypeOrder(QualType LHS, QualType RHS) const;
 
   /// Compare the rank of two floating point types as above, but compare equal
   /// if both types have the same floating-point semantics on the target (i.e.
   /// long double and double on AArch64 will return 0).
   int getFloatingTypeSemanticOrder(QualType LHS, QualType RHS) const;
 
   /// Return a real floating point or a complex type (based on
   /// \p typeDomain/\p typeSize).
   ///
   /// \param typeDomain a real floating point or complex type.
   /// \param typeSize a real floating point or complex type.
   QualType getFloatingTypeOfSizeWithinDomain(QualType typeSize,
                                              QualType typeDomain) const;
 
   unsigned getTargetAddressSpace(QualType T) const;
 
   unsigned getTargetAddressSpace(Qualifiers Q) const;
 
   unsigned getTargetAddressSpace(LangAS AS) const;
 
   LangAS getLangASForBuiltinAddressSpace(unsigned AS) const;
 
   /// Get target-dependent integer value for null pointer which is used for
   /// constant folding.
   uint64_t getTargetNullPointerValue(QualType QT) const;
 
   bool addressSpaceMapManglingFor(LangAS AS) const {
     return AddrSpaceMapMangling || isTargetAddressSpace(AS);
   }
 
 private:
   // Helper for integer ordering
   unsigned getIntegerRank(const Type *T) const;
 
 public:
   //===--------------------------------------------------------------------===//
   //                    Type Compatibility Predicates
   //===--------------------------------------------------------------------===//
 
   /// Compatibility predicates used to check assignment expressions.
   bool typesAreCompatible(QualType T1, QualType T2,
                           bool CompareUnqualified = false); // C99 6.2.7p1
 
   bool propertyTypesAreCompatible(QualType, QualType);
   bool typesAreBlockPointerCompatible(QualType, QualType);
 
   bool isObjCIdType(QualType T) const {
     return T == getObjCIdType();
   }
 
   bool isObjCClassType(QualType T) const {
     return T == getObjCClassType();
   }
 
   bool isObjCSelType(QualType T) const {
     return T == getObjCSelType();
   }
 
   bool ObjCQualifiedIdTypesAreCompatible(const ObjCObjectPointerType *LHS,
                                          const ObjCObjectPointerType *RHS,
                                          bool ForCompare);
 
   bool ObjCQualifiedClassTypesAreCompatible(const ObjCObjectPointerType *LHS,
                                             const ObjCObjectPointerType *RHS);
 
   // Check the safety of assignment from LHS to RHS
   bool canAssignObjCInterfaces(const ObjCObjectPointerType *LHSOPT,
                                const ObjCObjectPointerType *RHSOPT);
   bool canAssignObjCInterfaces(const ObjCObjectType *LHS,
                                const ObjCObjectType *RHS);
   bool canAssignObjCInterfacesInBlockPointer(
                                           const ObjCObjectPointerType *LHSOPT,
                                           const ObjCObjectPointerType *RHSOPT,
                                           bool BlockReturnType);
   bool areComparableObjCPointerTypes(QualType LHS, QualType RHS);
   QualType areCommonBaseCompatible(const ObjCObjectPointerType *LHSOPT,
                                    const ObjCObjectPointerType *RHSOPT);
   bool canBindObjCObjectType(QualType To, QualType From);
 
   // Functions for calculating composite types
   QualType mergeTypes(QualType, QualType, bool OfBlockPointer=false,
                       bool Unqualified = false, bool BlockReturnType = false);
   QualType mergeFunctionTypes(QualType, QualType, bool OfBlockPointer=false,
                               bool Unqualified = false, bool AllowCXX = false);
   QualType mergeFunctionParameterTypes(QualType, QualType,
                                        bool OfBlockPointer = false,
                                        bool Unqualified = false);
   QualType mergeTransparentUnionType(QualType, QualType,
                                      bool OfBlockPointer=false,
                                      bool Unqualified = false);
 
   QualType mergeObjCGCQualifiers(QualType, QualType);
 
   /// This function merges the ExtParameterInfo lists of two functions. It
   /// returns true if the lists are compatible. The merged list is returned in
   /// NewParamInfos.
   ///
   /// \param FirstFnType The type of the first function.
   ///
   /// \param SecondFnType The type of the second function.
   ///
   /// \param CanUseFirst This flag is set to true if the first function's
   /// ExtParameterInfo list can be used as the composite list of
   /// ExtParameterInfo.
   ///
   /// \param CanUseSecond This flag is set to true if the second function's
   /// ExtParameterInfo list can be used as the composite list of
   /// ExtParameterInfo.
   ///
   /// \param NewParamInfos The composite list of ExtParameterInfo. The list is
   /// empty if none of the flags are set.
   ///
   bool mergeExtParameterInfo(
       const FunctionProtoType *FirstFnType,
       const FunctionProtoType *SecondFnType,
       bool &CanUseFirst, bool &CanUseSecond,
       SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &NewParamInfos);
 
   void ResetObjCLayout(const ObjCContainerDecl *CD);
 
   //===--------------------------------------------------------------------===//
   //                    Integer Predicates
   //===--------------------------------------------------------------------===//
 
   // The width of an integer, as defined in C99 6.2.6.2. This is the number
   // of bits in an integer type excluding any padding bits.
   unsigned getIntWidth(QualType T) const;
 
   // Per C99 6.2.5p6, for every signed integer type, there is a corresponding
   // unsigned integer type.  This method takes a signed type, and returns the
   // corresponding unsigned integer type.
   // With the introduction of fixed point types in ISO N1169, this method also
   // accepts fixed point types and returns the corresponding unsigned type for
   // a given fixed point type.
   QualType getCorrespondingUnsignedType(QualType T) const;
 
   // Per C99 6.2.5p6, for every signed integer type, there is a corresponding
   // unsigned integer type.  This method takes an unsigned type, and returns the
   // corresponding signed integer type.
   // With the introduction of fixed point types in ISO N1169, this method also
   // accepts fixed point types and returns the corresponding signed type for
   // a given fixed point type.
   QualType getCorrespondingSignedType(QualType T) const;
 
   // Per ISO N1169, this method accepts fixed point types and returns the
   // corresponding saturated type for a given fixed point type.
   QualType getCorrespondingSaturatedType(QualType Ty) const;
 
   // This method accepts fixed point types and returns the corresponding signed
   // type. Unlike getCorrespondingUnsignedType(), this only accepts unsigned
   // fixed point types because there are unsigned integer types like bool and
   // char8_t that don't have signed equivalents.
   QualType getCorrespondingSignedFixedPointType(QualType Ty) const;
 
   //===--------------------------------------------------------------------===//
   //                    Integer Values
   //===--------------------------------------------------------------------===//
 
   /// Make an APSInt of the appropriate width and signedness for the
   /// given \p Value and integer \p Type.
   llvm::APSInt MakeIntValue(uint64_t Value, QualType Type) const {
     // If Type is a signed integer type larger than 64 bits, we need to be sure
     // to sign extend Res appropriately.
     llvm::APSInt Res(64, !Type->isSignedIntegerOrEnumerationType());
     Res = Value;
     unsigned Width = getIntWidth(Type);
     if (Width != Res.getBitWidth())
       return Res.extOrTrunc(Width);
     return Res;
   }
 
   bool isSentinelNullExpr(const Expr *E);
 
   /// Get the implementation of the ObjCInterfaceDecl \p D, or nullptr if
   /// none exists.
   ObjCImplementationDecl *getObjCImplementation(ObjCInterfaceDecl *D);
 
   /// Get the implementation of the ObjCCategoryDecl \p D, or nullptr if
   /// none exists.
   ObjCCategoryImplDecl *getObjCImplementation(ObjCCategoryDecl *D);
 
   /// Return true if there is at least one \@implementation in the TU.
   bool AnyObjCImplementation() {
     return !ObjCImpls.empty();
   }
 
   /// Set the implementation of ObjCInterfaceDecl.
   void setObjCImplementation(ObjCInterfaceDecl *IFaceD,
                              ObjCImplementationDecl *ImplD);
 
   /// Set the implementation of ObjCCategoryDecl.
   void setObjCImplementation(ObjCCategoryDecl *CatD,
                              ObjCCategoryImplDecl *ImplD);
 
   /// Get the duplicate declaration of a ObjCMethod in the same
   /// interface, or null if none exists.
   const ObjCMethodDecl *
   getObjCMethodRedeclaration(const ObjCMethodDecl *MD) const;
 
   void setObjCMethodRedeclaration(const ObjCMethodDecl *MD,
                                   const ObjCMethodDecl *Redecl);
 
   /// Returns the Objective-C interface that \p ND belongs to if it is
   /// an Objective-C method/property/ivar etc. that is part of an interface,
   /// otherwise returns null.
   const ObjCInterfaceDecl *getObjContainingInterface(const NamedDecl *ND) const;
 
   /// Set the copy initialization expression of a block var decl. \p CanThrow
   /// indicates whether the copy expression can throw or not.
   void setBlockVarCopyInit(const VarDecl* VD, Expr *CopyExpr, bool CanThrow);
 
   /// Get the copy initialization expression of the VarDecl \p VD, or
   /// nullptr if none exists.
   BlockVarCopyInit getBlockVarCopyInit(const VarDecl* VD) const;
 
   /// Allocate an uninitialized TypeSourceInfo.
   ///
   /// The caller should initialize the memory held by TypeSourceInfo using
   /// the TypeLoc wrappers.
   ///
   /// \param T the type that will be the basis for type source info. This type
   /// should refer to how the declarator was written in source code, not to
   /// what type semantic analysis resolved the declarator to.
   ///
   /// \param Size the size of the type info to create, or 0 if the size
   /// should be calculated based on the type.
   TypeSourceInfo *CreateTypeSourceInfo(QualType T, unsigned Size = 0) const;
 
   /// Allocate a TypeSourceInfo where all locations have been
   /// initialized to a given location, which defaults to the empty
   /// location.
   TypeSourceInfo *
   getTrivialTypeSourceInfo(QualType T,
                            SourceLocation Loc = SourceLocation()) const;
 
   /// Add a deallocation callback that will be invoked when the
   /// ASTContext is destroyed.
   ///
   /// \param Callback A callback function that will be invoked on destruction.
   ///
   /// \param Data Pointer data that will be provided to the callback function
   /// when it is called.
   void AddDeallocation(void (*Callback)(void *), void *Data) const;
 
   /// If T isn't trivially destructible, calls AddDeallocation to register it
   /// for destruction.
   template <typename T> void addDestruction(T *Ptr) const {
     if (!std::is_trivially_destructible<T>::value) {
       auto DestroyPtr = [](void *V) { static_cast<T *>(V)->~T(); };
       AddDeallocation(DestroyPtr, Ptr);
     }
   }
 
   GVALinkage GetGVALinkageForFunction(const FunctionDecl *FD) const;
   GVALinkage GetGVALinkageForVariable(const VarDecl *VD);
 
   /// Determines if the decl can be CodeGen'ed or deserialized from PCH
   /// lazily, only when used; this is only relevant for function or file scoped
   /// var definitions.
   ///
   /// \returns true if the function/var must be CodeGen'ed/deserialized even if
   /// it is not used.
   bool DeclMustBeEmitted(const Decl *D);
 
   /// Visits all versions of a multiversioned function with the passed
   /// predicate.
   void forEachMultiversionedFunctionVersion(
       const FunctionDecl *FD,
       llvm::function_ref<void(FunctionDecl *)> Pred) const;
 
   const CXXConstructorDecl *
   getCopyConstructorForExceptionObject(CXXRecordDecl *RD);
 
   void addCopyConstructorForExceptionObject(CXXRecordDecl *RD,
                                             CXXConstructorDecl *CD);
 
   void addTypedefNameForUnnamedTagDecl(TagDecl *TD, TypedefNameDecl *TND);
 
   TypedefNameDecl *getTypedefNameForUnnamedTagDecl(const TagDecl *TD);
 
   void addDeclaratorForUnnamedTagDecl(TagDecl *TD, DeclaratorDecl *DD);
 
   DeclaratorDecl *getDeclaratorForUnnamedTagDecl(const TagDecl *TD);
 
   void setManglingNumber(const NamedDecl *ND, unsigned Number);
   unsigned getManglingNumber(const NamedDecl *ND) const;
 
   void setStaticLocalNumber(const VarDecl *VD, unsigned Number);
   unsigned getStaticLocalNumber(const VarDecl *VD) const;
 
   /// Retrieve the context for computing mangling numbers in the given
   /// DeclContext.
   MangleNumberingContext &getManglingNumberContext(const DeclContext *DC);
   enum NeedExtraManglingDecl_t { NeedExtraManglingDecl };
   MangleNumberingContext &getManglingNumberContext(NeedExtraManglingDecl_t,
                                                    const Decl *D);
 
   std::unique_ptr<MangleNumberingContext> createMangleNumberingContext() const;
 
   /// Used by ParmVarDecl to store on the side the
   /// index of the parameter when it exceeds the size of the normal bitfield.
   void setParameterIndex(const ParmVarDecl *D, unsigned index);
 
   /// Used by ParmVarDecl to retrieve on the side the
   /// index of the parameter when it exceeds the size of the normal bitfield.
   unsigned getParameterIndex(const ParmVarDecl *D) const;
 
   /// Return a string representing the human readable name for the specified
   /// function declaration or file name. Used by SourceLocExpr and
   /// PredefinedExpr to cache evaluated results.
   StringLiteral *getPredefinedStringLiteralFromCache(StringRef Key) const;
 
   /// Return a declaration for the global GUID object representing the given
   /// GUID value.
   MSGuidDecl *getMSGuidDecl(MSGuidDeclParts Parts) const;
 
   /// Return the template parameter object of the given type with the given
   /// value.
   TemplateParamObjectDecl *getTemplateParamObjectDecl(QualType T,
                                                       const APValue &V) const;
 
   /// Parses the target attributes passed in, and returns only the ones that are
   /// valid feature names.
   ParsedTargetAttr filterFunctionTargetAttrs(const TargetAttr *TD) const;
 
   void getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap,
                              const FunctionDecl *) const;
   void getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap,
                              GlobalDecl GD) const;
 
   //===--------------------------------------------------------------------===//
   //                    Statistics
   //===--------------------------------------------------------------------===//
 
   /// The number of implicitly-declared default constructors.
   unsigned NumImplicitDefaultConstructors = 0;
 
   /// The number of implicitly-declared default constructors for
   /// which declarations were built.
   unsigned NumImplicitDefaultConstructorsDeclared = 0;
 
   /// The number of implicitly-declared copy constructors.
   unsigned NumImplicitCopyConstructors = 0;
 
   /// The number of implicitly-declared copy constructors for
   /// which declarations were built.
   unsigned NumImplicitCopyConstructorsDeclared = 0;
 
   /// The number of implicitly-declared move constructors.
   unsigned NumImplicitMoveConstructors = 0;
 
   /// The number of implicitly-declared move constructors for
   /// which declarations were built.
   unsigned NumImplicitMoveConstructorsDeclared = 0;
 
   /// The number of implicitly-declared copy assignment operators.
   unsigned NumImplicitCopyAssignmentOperators = 0;
 
   /// The number of implicitly-declared copy assignment operators for
   /// which declarations were built.
   unsigned NumImplicitCopyAssignmentOperatorsDeclared = 0;
 
   /// The number of implicitly-declared move assignment operators.
   unsigned NumImplicitMoveAssignmentOperators = 0;
 
   /// The number of implicitly-declared move assignment operators for
   /// which declarations were built.
   unsigned NumImplicitMoveAssignmentOperatorsDeclared = 0;
 
   /// The number of implicitly-declared destructors.
   unsigned NumImplicitDestructors = 0;
 
   /// The number of implicitly-declared destructors for which
   /// declarations were built.
   unsigned NumImplicitDestructorsDeclared = 0;
 
 public:
   /// Initialize built-in types.
   ///
   /// This routine may only be invoked once for a given ASTContext object.
   /// It is normally invoked after ASTContext construction.
   ///
   /// \param Target The target
   void InitBuiltinTypes(const TargetInfo &Target,
                         const TargetInfo *AuxTarget = nullptr);
 
 private:
   void InitBuiltinType(CanQualType &R, BuiltinType::Kind K);
 
   class ObjCEncOptions {
     unsigned Bits;
 
     ObjCEncOptions(unsigned Bits) : Bits(Bits) {}
 
   public:
     ObjCEncOptions() : Bits(0) {}
     ObjCEncOptions(const ObjCEncOptions &RHS) : Bits(RHS.Bits) {}
 
 #define OPT_LIST(V)                                                            \
   V(ExpandPointedToStructures, 0)                                              \
   V(ExpandStructures, 1)                                                       \
   V(IsOutermostType, 2)                                                        \
   V(EncodingProperty, 3)                                                       \
   V(IsStructField, 4)                                                          \
   V(EncodeBlockParameters, 5)                                                  \
   V(EncodeClassNames, 6)                                                       \
 
 #define V(N,I) ObjCEncOptions& set##N() { Bits |= 1 << I; return *this; }
 OPT_LIST(V)
 #undef V
 
 #define V(N,I) bool N() const { return Bits & 1 << I; }
 OPT_LIST(V)
 #undef V
 
 #undef OPT_LIST
 
     LLVM_NODISCARD ObjCEncOptions keepingOnly(ObjCEncOptions Mask) const {
       return Bits & Mask.Bits;
     }
 
     LLVM_NODISCARD ObjCEncOptions forComponentType() const {
       ObjCEncOptions Mask = ObjCEncOptions()
                                 .setIsOutermostType()
                                 .setIsStructField();
       return Bits & ~Mask.Bits;
     }
   };
 
   // Return the Objective-C type encoding for a given type.
   void getObjCEncodingForTypeImpl(QualType t, std::string &S,
                                   ObjCEncOptions Options,
                                   const FieldDecl *Field,
                                   QualType *NotEncodedT = nullptr) const;
 
   // Adds the encoding of the structure's members.
   void getObjCEncodingForStructureImpl(RecordDecl *RD, std::string &S,
                                        const FieldDecl *Field,
                                        bool includeVBases = true,
                                        QualType *NotEncodedT=nullptr) const;
 
 public:
   // Adds the encoding of a method parameter or return type.
   void getObjCEncodingForMethodParameter(Decl::ObjCDeclQualifier QT,
                                          QualType T, std::string& S,
                                          bool Extended) const;
 
   /// Returns true if this is an inline-initialized static data member
   /// which is treated as a definition for MSVC compatibility.
   bool isMSStaticDataMemberInlineDefinition(const VarDecl *VD) const;
 
   enum class InlineVariableDefinitionKind {
     /// Not an inline variable.
     None,
 
     /// Weak definition of inline variable.
     Weak,
 
     /// Weak for now, might become strong later in this TU.
     WeakUnknown,
 
     /// Strong definition.
     Strong
   };
 
   /// Determine whether a definition of this inline variable should
   /// be treated as a weak or strong definition. For compatibility with
   /// C++14 and before, for a constexpr static data member, if there is an
   /// out-of-line declaration of the member, we may promote it from weak to
   /// strong.
   InlineVariableDefinitionKind
   getInlineVariableDefinitionKind(const VarDecl *VD) const;
 
 private:
   friend class DeclarationNameTable;
   friend class DeclContext;
 
   const ASTRecordLayout &
   getObjCLayout(const ObjCInterfaceDecl *D,
                 const ObjCImplementationDecl *Impl) const;
 
   /// A set of deallocations that should be performed when the
   /// ASTContext is destroyed.
   // FIXME: We really should have a better mechanism in the ASTContext to
   // manage running destructors for types which do variable sized allocation
   // within the AST. In some places we thread the AST bump pointer allocator
   // into the datastructures which avoids this mess during deallocation but is
   // wasteful of memory, and here we require a lot of error prone book keeping
   // in order to track and run destructors while we're tearing things down.
   using DeallocationFunctionsAndArguments =
       llvm::SmallVector<std::pair<void (*)(void *), void *>, 16>;
   mutable DeallocationFunctionsAndArguments Deallocations;
 
   // FIXME: This currently contains the set of StoredDeclMaps used
   // by DeclContext objects.  This probably should not be in ASTContext,
   // but we include it here so that ASTContext can quickly deallocate them.
   llvm::PointerIntPair<StoredDeclsMap *, 1> LastSDM;
 
   std::vector<Decl *> TraversalScope;
 
   std::unique_ptr<VTableContextBase> VTContext;
 
   void ReleaseDeclContextMaps();
 
 public:
   enum PragmaSectionFlag : unsigned {
     PSF_None = 0,
     PSF_Read = 0x1,
     PSF_Write = 0x2,
     PSF_Execute = 0x4,
     PSF_Implicit = 0x8,
     PSF_ZeroInit = 0x10,
     PSF_Invalid = 0x80000000U,
   };
 
   struct SectionInfo {
     NamedDecl *Decl;
     SourceLocation PragmaSectionLocation;
     int SectionFlags;
 
     SectionInfo() = default;
     SectionInfo(NamedDecl *Decl, SourceLocation PragmaSectionLocation,
                 int SectionFlags)
         : Decl(Decl), PragmaSectionLocation(PragmaSectionLocation),
           SectionFlags(SectionFlags) {}
   };
 
   llvm::StringMap<SectionInfo> SectionInfos;
 
   /// Return a new OMPTraitInfo object owned by this context.
   OMPTraitInfo &getNewOMPTraitInfo();
 
-  /// Whether a C++ static variable may be externalized.
+  /// Whether a C++ static variable or CUDA/HIP kernel may be externalized.
   bool mayExternalizeStaticVar(const Decl *D) const;
 
-  /// Whether a C++ static variable should be externalized.
+  /// Whether a C++ static variable or CUDA/HIP kernel should be externalized.
   bool shouldExternalizeStaticVar(const Decl *D) const;
 
   StringRef getCUIDHash() const;
 
 private:
   /// All OMPTraitInfo objects live in this collection, one per
   /// `pragma omp [begin] declare variant` directive.
   SmallVector<std::unique_ptr<OMPTraitInfo>, 4> OMPTraitInfoVector;
 };
 
 /// Insertion operator for diagnostics.
 const StreamingDiagnostic &operator<<(const StreamingDiagnostic &DB,
                                       const ASTContext::SectionInfo &Section);
 
 /// Utility function for constructing a nullary selector.
 inline Selector GetNullarySelector(StringRef name, ASTContext &Ctx) {
   IdentifierInfo* II = &Ctx.Idents.get(name);
   return Ctx.Selectors.getSelector(0, &II);
 }
 
 /// Utility function for constructing an unary selector.
 inline Selector GetUnarySelector(StringRef name, ASTContext &Ctx) {
   IdentifierInfo* II = &Ctx.Idents.get(name);
   return Ctx.Selectors.getSelector(1, &II);
 }
 
 } // namespace clang
 
 // operator new and delete aren't allowed inside namespaces.
 
 /// Placement new for using the ASTContext's allocator.
 ///
 /// This placement form of operator new uses the ASTContext's allocator for
 /// obtaining memory.
 ///
 /// IMPORTANT: These are also declared in clang/AST/ASTContextAllocate.h!
 /// Any changes here need to also be made there.
 ///
 /// We intentionally avoid using a nothrow specification here so that the calls
 /// to this operator will not perform a null check on the result -- the
 /// underlying allocator never returns null pointers.
 ///
 /// Usage looks like this (assuming there's an ASTContext 'Context' in scope):
 /// @code
 /// // Default alignment (8)
 /// IntegerLiteral *Ex = new (Context) IntegerLiteral(arguments);
 /// // Specific alignment
 /// IntegerLiteral *Ex2 = new (Context, 4) IntegerLiteral(arguments);
 /// @endcode
 /// Memory allocated through this placement new operator does not need to be
 /// explicitly freed, as ASTContext will free all of this memory when it gets
 /// destroyed. Please note that you cannot use delete on the pointer.
 ///
 /// @param Bytes The number of bytes to allocate. Calculated by the compiler.
 /// @param C The ASTContext that provides the allocator.
 /// @param Alignment The alignment of the allocated memory (if the underlying
 ///                  allocator supports it).
 /// @return The allocated memory. Could be nullptr.
 inline void *operator new(size_t Bytes, const clang::ASTContext &C,
                           size_t Alignment /* = 8 */) {
   return C.Allocate(Bytes, Alignment);
 }
 
 /// Placement delete companion to the new above.
 ///
 /// This operator is just a companion to the new above. There is no way of
 /// invoking it directly; see the new operator for more details. This operator
 /// is called implicitly by the compiler if a placement new expression using
 /// the ASTContext throws in the object constructor.
 inline void operator delete(void *Ptr, const clang::ASTContext &C, size_t) {
   C.Deallocate(Ptr);
 }
 
 /// This placement form of operator new[] uses the ASTContext's allocator for
 /// obtaining memory.
 ///
 /// We intentionally avoid using a nothrow specification here so that the calls
 /// to this operator will not perform a null check on the result -- the
 /// underlying allocator never returns null pointers.
 ///
 /// Usage looks like this (assuming there's an ASTContext 'Context' in scope):
 /// @code
 /// // Default alignment (8)
 /// char *data = new (Context) char[10];
 /// // Specific alignment
 /// char *data = new (Context, 4) char[10];
 /// @endcode
 /// Memory allocated through this placement new[] operator does not need to be
 /// explicitly freed, as ASTContext will free all of this memory when it gets
 /// destroyed. Please note that you cannot use delete on the pointer.
 ///
 /// @param Bytes The number of bytes to allocate. Calculated by the compiler.
 /// @param C The ASTContext that provides the allocator.
 /// @param Alignment The alignment of the allocated memory (if the underlying
 ///                  allocator supports it).
 /// @return The allocated memory. Could be nullptr.
 inline void *operator new[](size_t Bytes, const clang::ASTContext& C,
                             size_t Alignment /* = 8 */) {
   return C.Allocate(Bytes, Alignment);
 }
 
 /// Placement delete[] companion to the new[] above.
 ///
 /// This operator is just a companion to the new[] above. There is no way of
 /// invoking it directly; see the new[] operator for more details. This operator
 /// is called implicitly by the compiler if a placement new[] expression using
 /// the ASTContext throws in the object constructor.
 inline void operator delete[](void *Ptr, const clang::ASTContext &C, size_t) {
   C.Deallocate(Ptr);
 }
 
 /// Create the representation of a LazyGenerationalUpdatePtr.
 template <typename Owner, typename T,
           void (clang::ExternalASTSource::*Update)(Owner)>
 typename clang::LazyGenerationalUpdatePtr<Owner, T, Update>::ValueType
     clang::LazyGenerationalUpdatePtr<Owner, T, Update>::makeValue(
         const clang::ASTContext &Ctx, T Value) {
   // Note, this is implemented here so that ExternalASTSource.h doesn't need to
   // include ASTContext.h. We explicitly instantiate it for all relevant types
   // in ASTContext.cpp.
   if (auto *Source = Ctx.getExternalSource())
     return new (Ctx) LazyData(Source, Value);
   return Value;
 }
 
 #endif // LLVM_CLANG_AST_ASTCONTEXT_H
diff --git a/contrib/llvm-project/clang/lib/AST/ASTContext.cpp b/contrib/llvm-project/clang/lib/AST/ASTContext.cpp
index 76c82f2e177c..e4b3827b8714 100644
--- a/contrib/llvm-project/clang/lib/AST/ASTContext.cpp
+++ b/contrib/llvm-project/clang/lib/AST/ASTContext.cpp
@@ -1,12285 +1,12288 @@
 //===- ASTContext.cpp - Context to hold long-lived AST nodes --------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //  This file implements the ASTContext interface.
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/ASTContext.h"
 #include "CXXABI.h"
 #include "Interp/Context.h"
 #include "clang/AST/APValue.h"
 #include "clang/AST/ASTConcept.h"
 #include "clang/AST/ASTMutationListener.h"
 #include "clang/AST/ASTTypeTraits.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/AttrIterator.h"
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/Comment.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclBase.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclContextInternals.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclOpenMP.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/DependenceFlags.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprConcepts.h"
 #include "clang/AST/ExternalASTSource.h"
 #include "clang/AST/Mangle.h"
 #include "clang/AST/MangleNumberingContext.h"
 #include "clang/AST/NestedNameSpecifier.h"
 #include "clang/AST/ParentMapContext.h"
 #include "clang/AST/RawCommentList.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/AST/Stmt.h"
 #include "clang/AST/TemplateBase.h"
 #include "clang/AST/TemplateName.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/AST/UnresolvedSet.h"
 #include "clang/AST/VTableBuilder.h"
 #include "clang/Basic/AddressSpaces.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/CommentOptions.h"
 #include "clang/Basic/ExceptionSpecificationType.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/Linkage.h"
 #include "clang/Basic/Module.h"
 #include "clang/Basic/NoSanitizeList.h"
 #include "clang/Basic/ObjCRuntime.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TargetCXXABI.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/XRayLists.h"
 #include "llvm/ADT/APFixedPoint.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/APSInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Support/Capacity.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
 #include <cstdlib>
 #include <map>
 #include <memory>
 #include <string>
 #include <tuple>
 #include <utility>
 
 using namespace clang;
 
 enum FloatingRank {
   BFloat16Rank,
   Float16Rank,
   HalfRank,
   FloatRank,
   DoubleRank,
   LongDoubleRank,
   Float128Rank,
   Ibm128Rank
 };
 
 /// \returns location that is relevant when searching for Doc comments related
 /// to \p D.
 static SourceLocation getDeclLocForCommentSearch(const Decl *D,
                                                  SourceManager &SourceMgr) {
   assert(D);
 
   // User can not attach documentation to implicit declarations.
   if (D->isImplicit())
     return {};
 
   // User can not attach documentation to implicit instantiations.
   if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
     if (FD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
       return {};
   }
 
   if (const auto *VD = dyn_cast<VarDecl>(D)) {
     if (VD->isStaticDataMember() &&
         VD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
       return {};
   }
 
   if (const auto *CRD = dyn_cast<CXXRecordDecl>(D)) {
     if (CRD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
       return {};
   }
 
   if (const auto *CTSD = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
     TemplateSpecializationKind TSK = CTSD->getSpecializationKind();
     if (TSK == TSK_ImplicitInstantiation ||
         TSK == TSK_Undeclared)
       return {};
   }
 
   if (const auto *ED = dyn_cast<EnumDecl>(D)) {
     if (ED->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
       return {};
   }
   if (const auto *TD = dyn_cast<TagDecl>(D)) {
     // When tag declaration (but not definition!) is part of the
     // decl-specifier-seq of some other declaration, it doesn't get comment
     if (TD->isEmbeddedInDeclarator() && !TD->isCompleteDefinition())
       return {};
   }
   // TODO: handle comments for function parameters properly.
   if (isa<ParmVarDecl>(D))
     return {};
 
   // TODO: we could look up template parameter documentation in the template
   // documentation.
   if (isa<TemplateTypeParmDecl>(D) ||
       isa<NonTypeTemplateParmDecl>(D) ||
       isa<TemplateTemplateParmDecl>(D))
     return {};
 
   // Find declaration location.
   // For Objective-C declarations we generally don't expect to have multiple
   // declarators, thus use declaration starting location as the "declaration
   // location".
   // For all other declarations multiple declarators are used quite frequently,
   // so we use the location of the identifier as the "declaration location".
   if (isa<ObjCMethodDecl>(D) || isa<ObjCContainerDecl>(D) ||
       isa<ObjCPropertyDecl>(D) ||
       isa<RedeclarableTemplateDecl>(D) ||
       isa<ClassTemplateSpecializationDecl>(D) ||
       // Allow association with Y across {} in `typedef struct X {} Y`.
       isa<TypedefDecl>(D))
     return D->getBeginLoc();
 
   const SourceLocation DeclLoc = D->getLocation();
   if (DeclLoc.isMacroID()) {
     if (isa<TypedefDecl>(D)) {
       // If location of the typedef name is in a macro, it is because being
       // declared via a macro. Try using declaration's starting location as
       // the "declaration location".
       return D->getBeginLoc();
     }
 
     if (const auto *TD = dyn_cast<TagDecl>(D)) {
       // If location of the tag decl is inside a macro, but the spelling of
       // the tag name comes from a macro argument, it looks like a special
       // macro like NS_ENUM is being used to define the tag decl.  In that
       // case, adjust the source location to the expansion loc so that we can
       // attach the comment to the tag decl.
       if (SourceMgr.isMacroArgExpansion(DeclLoc) && TD->isCompleteDefinition())
         return SourceMgr.getExpansionLoc(DeclLoc);
     }
   }
 
   return DeclLoc;
 }
 
 RawComment *ASTContext::getRawCommentForDeclNoCacheImpl(
     const Decl *D, const SourceLocation RepresentativeLocForDecl,
     const std::map<unsigned, RawComment *> &CommentsInTheFile) const {
   // If the declaration doesn't map directly to a location in a file, we
   // can't find the comment.
   if (RepresentativeLocForDecl.isInvalid() ||
       !RepresentativeLocForDecl.isFileID())
     return nullptr;
 
   // If there are no comments anywhere, we won't find anything.
   if (CommentsInTheFile.empty())
     return nullptr;
 
   // Decompose the location for the declaration and find the beginning of the
   // file buffer.
   const std::pair<FileID, unsigned> DeclLocDecomp =
       SourceMgr.getDecomposedLoc(RepresentativeLocForDecl);
 
   // Slow path.
   auto OffsetCommentBehindDecl =
       CommentsInTheFile.lower_bound(DeclLocDecomp.second);
 
   // First check whether we have a trailing comment.
   if (OffsetCommentBehindDecl != CommentsInTheFile.end()) {
     RawComment *CommentBehindDecl = OffsetCommentBehindDecl->second;
     if ((CommentBehindDecl->isDocumentation() ||
          LangOpts.CommentOpts.ParseAllComments) &&
         CommentBehindDecl->isTrailingComment() &&
         (isa<FieldDecl>(D) || isa<EnumConstantDecl>(D) || isa<VarDecl>(D) ||
          isa<ObjCMethodDecl>(D) || isa<ObjCPropertyDecl>(D))) {
 
       // Check that Doxygen trailing comment comes after the declaration, starts
       // on the same line and in the same file as the declaration.
       if (SourceMgr.getLineNumber(DeclLocDecomp.first, DeclLocDecomp.second) ==
           Comments.getCommentBeginLine(CommentBehindDecl, DeclLocDecomp.first,
                                        OffsetCommentBehindDecl->first)) {
         return CommentBehindDecl;
       }
     }
   }
 
   // The comment just after the declaration was not a trailing comment.
   // Let's look at the previous comment.
   if (OffsetCommentBehindDecl == CommentsInTheFile.begin())
     return nullptr;
 
   auto OffsetCommentBeforeDecl = --OffsetCommentBehindDecl;
   RawComment *CommentBeforeDecl = OffsetCommentBeforeDecl->second;
 
   // Check that we actually have a non-member Doxygen comment.
   if (!(CommentBeforeDecl->isDocumentation() ||
         LangOpts.CommentOpts.ParseAllComments) ||
       CommentBeforeDecl->isTrailingComment())
     return nullptr;
 
   // Decompose the end of the comment.
   const unsigned CommentEndOffset =
       Comments.getCommentEndOffset(CommentBeforeDecl);
 
   // Get the corresponding buffer.
   bool Invalid = false;
   const char *Buffer = SourceMgr.getBufferData(DeclLocDecomp.first,
                                                &Invalid).data();
   if (Invalid)
     return nullptr;
 
   // Extract text between the comment and declaration.
   StringRef Text(Buffer + CommentEndOffset,
                  DeclLocDecomp.second - CommentEndOffset);
 
   // There should be no other declarations or preprocessor directives between
   // comment and declaration.
   if (Text.find_first_of(";{}#@") != StringRef::npos)
     return nullptr;
 
   return CommentBeforeDecl;
 }
 
 RawComment *ASTContext::getRawCommentForDeclNoCache(const Decl *D) const {
   const SourceLocation DeclLoc = getDeclLocForCommentSearch(D, SourceMgr);
 
   // If the declaration doesn't map directly to a location in a file, we
   // can't find the comment.
   if (DeclLoc.isInvalid() || !DeclLoc.isFileID())
     return nullptr;
 
   if (ExternalSource && !CommentsLoaded) {
     ExternalSource->ReadComments();
     CommentsLoaded = true;
   }
 
   if (Comments.empty())
     return nullptr;
 
   const FileID File = SourceMgr.getDecomposedLoc(DeclLoc).first;
   const auto CommentsInThisFile = Comments.getCommentsInFile(File);
   if (!CommentsInThisFile || CommentsInThisFile->empty())
     return nullptr;
 
   return getRawCommentForDeclNoCacheImpl(D, DeclLoc, *CommentsInThisFile);
 }
 
 void ASTContext::addComment(const RawComment &RC) {
   assert(LangOpts.RetainCommentsFromSystemHeaders ||
          !SourceMgr.isInSystemHeader(RC.getSourceRange().getBegin()));
   Comments.addComment(RC, LangOpts.CommentOpts, BumpAlloc);
 }
 
 /// If we have a 'templated' declaration for a template, adjust 'D' to
 /// refer to the actual template.
 /// If we have an implicit instantiation, adjust 'D' to refer to template.
 static const Decl &adjustDeclToTemplate(const Decl &D) {
   if (const auto *FD = dyn_cast<FunctionDecl>(&D)) {
     // Is this function declaration part of a function template?
     if (const FunctionTemplateDecl *FTD = FD->getDescribedFunctionTemplate())
       return *FTD;
 
     // Nothing to do if function is not an implicit instantiation.
     if (FD->getTemplateSpecializationKind() != TSK_ImplicitInstantiation)
       return D;
 
     // Function is an implicit instantiation of a function template?
     if (const FunctionTemplateDecl *FTD = FD->getPrimaryTemplate())
       return *FTD;
 
     // Function is instantiated from a member definition of a class template?
     if (const FunctionDecl *MemberDecl =
             FD->getInstantiatedFromMemberFunction())
       return *MemberDecl;
 
     return D;
   }
   if (const auto *VD = dyn_cast<VarDecl>(&D)) {
     // Static data member is instantiated from a member definition of a class
     // template?
     if (VD->isStaticDataMember())
       if (const VarDecl *MemberDecl = VD->getInstantiatedFromStaticDataMember())
         return *MemberDecl;
 
     return D;
   }
   if (const auto *CRD = dyn_cast<CXXRecordDecl>(&D)) {
     // Is this class declaration part of a class template?
     if (const ClassTemplateDecl *CTD = CRD->getDescribedClassTemplate())
       return *CTD;
 
     // Class is an implicit instantiation of a class template or partial
     // specialization?
     if (const auto *CTSD = dyn_cast<ClassTemplateSpecializationDecl>(CRD)) {
       if (CTSD->getSpecializationKind() != TSK_ImplicitInstantiation)
         return D;
       llvm::PointerUnion<ClassTemplateDecl *,
                          ClassTemplatePartialSpecializationDecl *>
           PU = CTSD->getSpecializedTemplateOrPartial();
       return PU.is<ClassTemplateDecl *>()
                  ? *static_cast<const Decl *>(PU.get<ClassTemplateDecl *>())
                  : *static_cast<const Decl *>(
                        PU.get<ClassTemplatePartialSpecializationDecl *>());
     }
 
     // Class is instantiated from a member definition of a class template?
     if (const MemberSpecializationInfo *Info =
             CRD->getMemberSpecializationInfo())
       return *Info->getInstantiatedFrom();
 
     return D;
   }
   if (const auto *ED = dyn_cast<EnumDecl>(&D)) {
     // Enum is instantiated from a member definition of a class template?
     if (const EnumDecl *MemberDecl = ED->getInstantiatedFromMemberEnum())
       return *MemberDecl;
 
     return D;
   }
   // FIXME: Adjust alias templates?
   return D;
 }
 
 const RawComment *ASTContext::getRawCommentForAnyRedecl(
                                                 const Decl *D,
                                                 const Decl **OriginalDecl) const {
   if (!D) {
     if (OriginalDecl)
       OriginalDecl = nullptr;
     return nullptr;
   }
 
   D = &adjustDeclToTemplate(*D);
 
   // Any comment directly attached to D?
   {
     auto DeclComment = DeclRawComments.find(D);
     if (DeclComment != DeclRawComments.end()) {
       if (OriginalDecl)
         *OriginalDecl = D;
       return DeclComment->second;
     }
   }
 
   // Any comment attached to any redeclaration of D?
   const Decl *CanonicalD = D->getCanonicalDecl();
   if (!CanonicalD)
     return nullptr;
 
   {
     auto RedeclComment = RedeclChainComments.find(CanonicalD);
     if (RedeclComment != RedeclChainComments.end()) {
       if (OriginalDecl)
         *OriginalDecl = RedeclComment->second;
       auto CommentAtRedecl = DeclRawComments.find(RedeclComment->second);
       assert(CommentAtRedecl != DeclRawComments.end() &&
              "This decl is supposed to have comment attached.");
       return CommentAtRedecl->second;
     }
   }
 
   // Any redeclarations of D that we haven't checked for comments yet?
   // We can't use DenseMap::iterator directly since it'd get invalid.
   auto LastCheckedRedecl = [this, CanonicalD]() -> const Decl * {
     auto LookupRes = CommentlessRedeclChains.find(CanonicalD);
     if (LookupRes != CommentlessRedeclChains.end())
       return LookupRes->second;
     return nullptr;
   }();
 
   for (const auto Redecl : D->redecls()) {
     assert(Redecl);
     // Skip all redeclarations that have been checked previously.
     if (LastCheckedRedecl) {
       if (LastCheckedRedecl == Redecl) {
         LastCheckedRedecl = nullptr;
       }
       continue;
     }
     const RawComment *RedeclComment = getRawCommentForDeclNoCache(Redecl);
     if (RedeclComment) {
       cacheRawCommentForDecl(*Redecl, *RedeclComment);
       if (OriginalDecl)
         *OriginalDecl = Redecl;
       return RedeclComment;
     }
     CommentlessRedeclChains[CanonicalD] = Redecl;
   }
 
   if (OriginalDecl)
     *OriginalDecl = nullptr;
   return nullptr;
 }
 
 void ASTContext::cacheRawCommentForDecl(const Decl &OriginalD,
                                         const RawComment &Comment) const {
   assert(Comment.isDocumentation() || LangOpts.CommentOpts.ParseAllComments);
   DeclRawComments.try_emplace(&OriginalD, &Comment);
   const Decl *const CanonicalDecl = OriginalD.getCanonicalDecl();
   RedeclChainComments.try_emplace(CanonicalDecl, &OriginalD);
   CommentlessRedeclChains.erase(CanonicalDecl);
 }
 
 static void addRedeclaredMethods(const ObjCMethodDecl *ObjCMethod,
                    SmallVectorImpl<const NamedDecl *> &Redeclared) {
   const DeclContext *DC = ObjCMethod->getDeclContext();
   if (const auto *IMD = dyn_cast<ObjCImplDecl>(DC)) {
     const ObjCInterfaceDecl *ID = IMD->getClassInterface();
     if (!ID)
       return;
     // Add redeclared method here.
     for (const auto *Ext : ID->known_extensions()) {
       if (ObjCMethodDecl *RedeclaredMethod =
             Ext->getMethod(ObjCMethod->getSelector(),
                                   ObjCMethod->isInstanceMethod()))
         Redeclared.push_back(RedeclaredMethod);
     }
   }
 }
 
 void ASTContext::attachCommentsToJustParsedDecls(ArrayRef<Decl *> Decls,
                                                  const Preprocessor *PP) {
   if (Comments.empty() || Decls.empty())
     return;
 
   FileID File;
   for (Decl *D : Decls) {
     SourceLocation Loc = D->getLocation();
     if (Loc.isValid()) {
       // See if there are any new comments that are not attached to a decl.
       // The location doesn't have to be precise - we care only about the file.
       File = SourceMgr.getDecomposedLoc(Loc).first;
       break;
     }
   }
 
   if (File.isInvalid())
     return;
 
   auto CommentsInThisFile = Comments.getCommentsInFile(File);
   if (!CommentsInThisFile || CommentsInThisFile->empty() ||
       CommentsInThisFile->rbegin()->second->isAttached())
     return;
 
   // There is at least one comment not attached to a decl.
   // Maybe it should be attached to one of Decls?
   //
   // Note that this way we pick up not only comments that precede the
   // declaration, but also comments that *follow* the declaration -- thanks to
   // the lookahead in the lexer: we've consumed the semicolon and looked
   // ahead through comments.
 
   for (const Decl *D : Decls) {
     assert(D);
     if (D->isInvalidDecl())
       continue;
 
     D = &adjustDeclToTemplate(*D);
 
     const SourceLocation DeclLoc = getDeclLocForCommentSearch(D, SourceMgr);
 
     if (DeclLoc.isInvalid() || !DeclLoc.isFileID())
       continue;
 
     if (DeclRawComments.count(D) > 0)
       continue;
 
     if (RawComment *const DocComment =
             getRawCommentForDeclNoCacheImpl(D, DeclLoc, *CommentsInThisFile)) {
       cacheRawCommentForDecl(*D, *DocComment);
       comments::FullComment *FC = DocComment->parse(*this, PP, D);
       ParsedComments[D->getCanonicalDecl()] = FC;
     }
   }
 }
 
 comments::FullComment *ASTContext::cloneFullComment(comments::FullComment *FC,
                                                     const Decl *D) const {
   auto *ThisDeclInfo = new (*this) comments::DeclInfo;
   ThisDeclInfo->CommentDecl = D;
   ThisDeclInfo->IsFilled = false;
   ThisDeclInfo->fill();
   ThisDeclInfo->CommentDecl = FC->getDecl();
   if (!ThisDeclInfo->TemplateParameters)
     ThisDeclInfo->TemplateParameters = FC->getDeclInfo()->TemplateParameters;
   comments::FullComment *CFC =
     new (*this) comments::FullComment(FC->getBlocks(),
                                       ThisDeclInfo);
   return CFC;
 }
 
 comments::FullComment *ASTContext::getLocalCommentForDeclUncached(const Decl *D) const {
   const RawComment *RC = getRawCommentForDeclNoCache(D);
   return RC ? RC->parse(*this, nullptr, D) : nullptr;
 }
 
 comments::FullComment *ASTContext::getCommentForDecl(
                                               const Decl *D,
                                               const Preprocessor *PP) const {
   if (!D || D->isInvalidDecl())
     return nullptr;
   D = &adjustDeclToTemplate(*D);
 
   const Decl *Canonical = D->getCanonicalDecl();
   llvm::DenseMap<const Decl *, comments::FullComment *>::iterator Pos =
       ParsedComments.find(Canonical);
 
   if (Pos != ParsedComments.end()) {
     if (Canonical != D) {
       comments::FullComment *FC = Pos->second;
       comments::FullComment *CFC = cloneFullComment(FC, D);
       return CFC;
     }
     return Pos->second;
   }
 
   const Decl *OriginalDecl = nullptr;
 
   const RawComment *RC = getRawCommentForAnyRedecl(D, &OriginalDecl);
   if (!RC) {
     if (isa<ObjCMethodDecl>(D) || isa<FunctionDecl>(D)) {
       SmallVector<const NamedDecl*, 8> Overridden;
       const auto *OMD = dyn_cast<ObjCMethodDecl>(D);
       if (OMD && OMD->isPropertyAccessor())
         if (const ObjCPropertyDecl *PDecl = OMD->findPropertyDecl())
           if (comments::FullComment *FC = getCommentForDecl(PDecl, PP))
             return cloneFullComment(FC, D);
       if (OMD)
         addRedeclaredMethods(OMD, Overridden);
       getOverriddenMethods(dyn_cast<NamedDecl>(D), Overridden);
       for (unsigned i = 0, e = Overridden.size(); i < e; i++)
         if (comments::FullComment *FC = getCommentForDecl(Overridden[i], PP))
           return cloneFullComment(FC, D);
     }
     else if (const auto *TD = dyn_cast<TypedefNameDecl>(D)) {
       // Attach any tag type's documentation to its typedef if latter
       // does not have one of its own.
       QualType QT = TD->getUnderlyingType();
       if (const auto *TT = QT->getAs<TagType>())
         if (const Decl *TD = TT->getDecl())
           if (comments::FullComment *FC = getCommentForDecl(TD, PP))
             return cloneFullComment(FC, D);
     }
     else if (const auto *IC = dyn_cast<ObjCInterfaceDecl>(D)) {
       while (IC->getSuperClass()) {
         IC = IC->getSuperClass();
         if (comments::FullComment *FC = getCommentForDecl(IC, PP))
           return cloneFullComment(FC, D);
       }
     }
     else if (const auto *CD = dyn_cast<ObjCCategoryDecl>(D)) {
       if (const ObjCInterfaceDecl *IC = CD->getClassInterface())
         if (comments::FullComment *FC = getCommentForDecl(IC, PP))
           return cloneFullComment(FC, D);
     }
     else if (const auto *RD = dyn_cast<CXXRecordDecl>(D)) {
       if (!(RD = RD->getDefinition()))
         return nullptr;
       // Check non-virtual bases.
       for (const auto &I : RD->bases()) {
         if (I.isVirtual() || (I.getAccessSpecifier() != AS_public))
           continue;
         QualType Ty = I.getType();
         if (Ty.isNull())
           continue;
         if (const CXXRecordDecl *NonVirtualBase = Ty->getAsCXXRecordDecl()) {
           if (!(NonVirtualBase= NonVirtualBase->getDefinition()))
             continue;
 
           if (comments::FullComment *FC = getCommentForDecl((NonVirtualBase), PP))
             return cloneFullComment(FC, D);
         }
       }
       // Check virtual bases.
       for (const auto &I : RD->vbases()) {
         if (I.getAccessSpecifier() != AS_public)
           continue;
         QualType Ty = I.getType();
         if (Ty.isNull())
           continue;
         if (const CXXRecordDecl *VirtualBase = Ty->getAsCXXRecordDecl()) {
           if (!(VirtualBase= VirtualBase->getDefinition()))
             continue;
           if (comments::FullComment *FC = getCommentForDecl((VirtualBase), PP))
             return cloneFullComment(FC, D);
         }
       }
     }
     return nullptr;
   }
 
   // If the RawComment was attached to other redeclaration of this Decl, we
   // should parse the comment in context of that other Decl.  This is important
   // because comments can contain references to parameter names which can be
   // different across redeclarations.
   if (D != OriginalDecl && OriginalDecl)
     return getCommentForDecl(OriginalDecl, PP);
 
   comments::FullComment *FC = RC->parse(*this, PP, D);
   ParsedComments[Canonical] = FC;
   return FC;
 }
 
 void
 ASTContext::CanonicalTemplateTemplateParm::Profile(llvm::FoldingSetNodeID &ID,
                                                    const ASTContext &C,
                                                TemplateTemplateParmDecl *Parm) {
   ID.AddInteger(Parm->getDepth());
   ID.AddInteger(Parm->getPosition());
   ID.AddBoolean(Parm->isParameterPack());
 
   TemplateParameterList *Params = Parm->getTemplateParameters();
   ID.AddInteger(Params->size());
   for (TemplateParameterList::const_iterator P = Params->begin(),
                                           PEnd = Params->end();
        P != PEnd; ++P) {
     if (const auto *TTP = dyn_cast<TemplateTypeParmDecl>(*P)) {
       ID.AddInteger(0);
       ID.AddBoolean(TTP->isParameterPack());
       const TypeConstraint *TC = TTP->getTypeConstraint();
       ID.AddBoolean(TC != nullptr);
       if (TC)
         TC->getImmediatelyDeclaredConstraint()->Profile(ID, C,
                                                         /*Canonical=*/true);
       if (TTP->isExpandedParameterPack()) {
         ID.AddBoolean(true);
         ID.AddInteger(TTP->getNumExpansionParameters());
       } else
         ID.AddBoolean(false);
       continue;
     }
 
     if (const auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(*P)) {
       ID.AddInteger(1);
       ID.AddBoolean(NTTP->isParameterPack());
       ID.AddPointer(NTTP->getType().getCanonicalType().getAsOpaquePtr());
       if (NTTP->isExpandedParameterPack()) {
         ID.AddBoolean(true);
         ID.AddInteger(NTTP->getNumExpansionTypes());
         for (unsigned I = 0, N = NTTP->getNumExpansionTypes(); I != N; ++I) {
           QualType T = NTTP->getExpansionType(I);
           ID.AddPointer(T.getCanonicalType().getAsOpaquePtr());
         }
       } else
         ID.AddBoolean(false);
       continue;
     }
 
     auto *TTP = cast<TemplateTemplateParmDecl>(*P);
     ID.AddInteger(2);
     Profile(ID, C, TTP);
   }
   Expr *RequiresClause = Parm->getTemplateParameters()->getRequiresClause();
   ID.AddBoolean(RequiresClause != nullptr);
   if (RequiresClause)
     RequiresClause->Profile(ID, C, /*Canonical=*/true);
 }
 
 static Expr *
 canonicalizeImmediatelyDeclaredConstraint(const ASTContext &C, Expr *IDC,
                                           QualType ConstrainedType) {
   // This is a bit ugly - we need to form a new immediately-declared
   // constraint that references the new parameter; this would ideally
   // require semantic analysis (e.g. template<C T> struct S {}; - the
   // converted arguments of C<T> could be an argument pack if C is
   // declared as template<typename... T> concept C = ...).
   // We don't have semantic analysis here so we dig deep into the
   // ready-made constraint expr and change the thing manually.
   ConceptSpecializationExpr *CSE;
   if (const auto *Fold = dyn_cast<CXXFoldExpr>(IDC))
     CSE = cast<ConceptSpecializationExpr>(Fold->getLHS());
   else
     CSE = cast<ConceptSpecializationExpr>(IDC);
   ArrayRef<TemplateArgument> OldConverted = CSE->getTemplateArguments();
   SmallVector<TemplateArgument, 3> NewConverted;
   NewConverted.reserve(OldConverted.size());
   if (OldConverted.front().getKind() == TemplateArgument::Pack) {
     // The case:
     // template<typename... T> concept C = true;
     // template<C<int> T> struct S; -> constraint is C<{T, int}>
     NewConverted.push_back(ConstrainedType);
     for (auto &Arg : OldConverted.front().pack_elements().drop_front(1))
       NewConverted.push_back(Arg);
     TemplateArgument NewPack(NewConverted);
 
     NewConverted.clear();
     NewConverted.push_back(NewPack);
     assert(OldConverted.size() == 1 &&
            "Template parameter pack should be the last parameter");
   } else {
     assert(OldConverted.front().getKind() == TemplateArgument::Type &&
            "Unexpected first argument kind for immediately-declared "
            "constraint");
     NewConverted.push_back(ConstrainedType);
     for (auto &Arg : OldConverted.drop_front(1))
       NewConverted.push_back(Arg);
   }
   Expr *NewIDC = ConceptSpecializationExpr::Create(
       C, CSE->getNamedConcept(), NewConverted, nullptr,
       CSE->isInstantiationDependent(), CSE->containsUnexpandedParameterPack());
 
   if (auto *OrigFold = dyn_cast<CXXFoldExpr>(IDC))
     NewIDC = new (C) CXXFoldExpr(
         OrigFold->getType(), /*Callee*/nullptr, SourceLocation(), NewIDC,
         BinaryOperatorKind::BO_LAnd, SourceLocation(), /*RHS=*/nullptr,
         SourceLocation(), /*NumExpansions=*/None);
   return NewIDC;
 }
 
 TemplateTemplateParmDecl *
 ASTContext::getCanonicalTemplateTemplateParmDecl(
                                           TemplateTemplateParmDecl *TTP) const {
   // Check if we already have a canonical template template parameter.
   llvm::FoldingSetNodeID ID;
   CanonicalTemplateTemplateParm::Profile(ID, *this, TTP);
   void *InsertPos = nullptr;
   CanonicalTemplateTemplateParm *Canonical
     = CanonTemplateTemplateParms.FindNodeOrInsertPos(ID, InsertPos);
   if (Canonical)
     return Canonical->getParam();
 
   // Build a canonical template parameter list.
   TemplateParameterList *Params = TTP->getTemplateParameters();
   SmallVector<NamedDecl *, 4> CanonParams;
   CanonParams.reserve(Params->size());
   for (TemplateParameterList::const_iterator P = Params->begin(),
                                           PEnd = Params->end();
        P != PEnd; ++P) {
     if (const auto *TTP = dyn_cast<TemplateTypeParmDecl>(*P)) {
       TemplateTypeParmDecl *NewTTP = TemplateTypeParmDecl::Create(*this,
           getTranslationUnitDecl(), SourceLocation(), SourceLocation(),
           TTP->getDepth(), TTP->getIndex(), nullptr, false,
           TTP->isParameterPack(), TTP->hasTypeConstraint(),
           TTP->isExpandedParameterPack() ?
           llvm::Optional<unsigned>(TTP->getNumExpansionParameters()) : None);
       if (const auto *TC = TTP->getTypeConstraint()) {
         QualType ParamAsArgument(NewTTP->getTypeForDecl(), 0);
         Expr *NewIDC = canonicalizeImmediatelyDeclaredConstraint(
                 *this, TC->getImmediatelyDeclaredConstraint(),
                 ParamAsArgument);
         TemplateArgumentListInfo CanonArgsAsWritten;
         if (auto *Args = TC->getTemplateArgsAsWritten())
           for (const auto &ArgLoc : Args->arguments())
             CanonArgsAsWritten.addArgument(
                 TemplateArgumentLoc(ArgLoc.getArgument(),
                                     TemplateArgumentLocInfo()));
         NewTTP->setTypeConstraint(
             NestedNameSpecifierLoc(),
             DeclarationNameInfo(TC->getNamedConcept()->getDeclName(),
                                 SourceLocation()), /*FoundDecl=*/nullptr,
             // Actually canonicalizing a TemplateArgumentLoc is difficult so we
             // simply omit the ArgsAsWritten
             TC->getNamedConcept(), /*ArgsAsWritten=*/nullptr, NewIDC);
       }
       CanonParams.push_back(NewTTP);
     } else if (const auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(*P)) {
       QualType T = getCanonicalType(NTTP->getType());
       TypeSourceInfo *TInfo = getTrivialTypeSourceInfo(T);
       NonTypeTemplateParmDecl *Param;
       if (NTTP->isExpandedParameterPack()) {
         SmallVector<QualType, 2> ExpandedTypes;
         SmallVector<TypeSourceInfo *, 2> ExpandedTInfos;
         for (unsigned I = 0, N = NTTP->getNumExpansionTypes(); I != N; ++I) {
           ExpandedTypes.push_back(getCanonicalType(NTTP->getExpansionType(I)));
           ExpandedTInfos.push_back(
                                 getTrivialTypeSourceInfo(ExpandedTypes.back()));
         }
 
         Param = NonTypeTemplateParmDecl::Create(*this, getTranslationUnitDecl(),
                                                 SourceLocation(),
                                                 SourceLocation(),
                                                 NTTP->getDepth(),
                                                 NTTP->getPosition(), nullptr,
                                                 T,
                                                 TInfo,
                                                 ExpandedTypes,
                                                 ExpandedTInfos);
       } else {
         Param = NonTypeTemplateParmDecl::Create(*this, getTranslationUnitDecl(),
                                                 SourceLocation(),
                                                 SourceLocation(),
                                                 NTTP->getDepth(),
                                                 NTTP->getPosition(), nullptr,
                                                 T,
                                                 NTTP->isParameterPack(),
                                                 TInfo);
       }
       if (AutoType *AT = T->getContainedAutoType()) {
         if (AT->isConstrained()) {
           Param->setPlaceholderTypeConstraint(
               canonicalizeImmediatelyDeclaredConstraint(
                   *this, NTTP->getPlaceholderTypeConstraint(), T));
         }
       }
       CanonParams.push_back(Param);
 
     } else
       CanonParams.push_back(getCanonicalTemplateTemplateParmDecl(
                                            cast<TemplateTemplateParmDecl>(*P)));
   }
 
   Expr *CanonRequiresClause = nullptr;
   if (Expr *RequiresClause = TTP->getTemplateParameters()->getRequiresClause())
     CanonRequiresClause = RequiresClause;
 
   TemplateTemplateParmDecl *CanonTTP
     = TemplateTemplateParmDecl::Create(*this, getTranslationUnitDecl(),
                                        SourceLocation(), TTP->getDepth(),
                                        TTP->getPosition(),
                                        TTP->isParameterPack(),
                                        nullptr,
                          TemplateParameterList::Create(*this, SourceLocation(),
                                                        SourceLocation(),
                                                        CanonParams,
                                                        SourceLocation(),
                                                        CanonRequiresClause));
 
   // Get the new insert position for the node we care about.
   Canonical = CanonTemplateTemplateParms.FindNodeOrInsertPos(ID, InsertPos);
   assert(!Canonical && "Shouldn't be in the map!");
   (void)Canonical;
 
   // Create the canonical template template parameter entry.
   Canonical = new (*this) CanonicalTemplateTemplateParm(CanonTTP);
   CanonTemplateTemplateParms.InsertNode(Canonical, InsertPos);
   return CanonTTP;
 }
 
 TargetCXXABI::Kind ASTContext::getCXXABIKind() const {
   auto Kind = getTargetInfo().getCXXABI().getKind();
   return getLangOpts().CXXABI.getValueOr(Kind);
 }
 
 CXXABI *ASTContext::createCXXABI(const TargetInfo &T) {
   if (!LangOpts.CPlusPlus) return nullptr;
 
   switch (getCXXABIKind()) {
   case TargetCXXABI::AppleARM64:
   case TargetCXXABI::Fuchsia:
   case TargetCXXABI::GenericARM: // Same as Itanium at this level
   case TargetCXXABI::iOS:
   case TargetCXXABI::WatchOS:
   case TargetCXXABI::GenericAArch64:
   case TargetCXXABI::GenericMIPS:
   case TargetCXXABI::GenericItanium:
   case TargetCXXABI::WebAssembly:
   case TargetCXXABI::XL:
     return CreateItaniumCXXABI(*this);
   case TargetCXXABI::Microsoft:
     return CreateMicrosoftCXXABI(*this);
   }
   llvm_unreachable("Invalid CXXABI type!");
 }
 
 interp::Context &ASTContext::getInterpContext() {
   if (!InterpContext) {
     InterpContext.reset(new interp::Context(*this));
   }
   return *InterpContext.get();
 }
 
 ParentMapContext &ASTContext::getParentMapContext() {
   if (!ParentMapCtx)
     ParentMapCtx.reset(new ParentMapContext(*this));
   return *ParentMapCtx.get();
 }
 
 static const LangASMap *getAddressSpaceMap(const TargetInfo &T,
                                            const LangOptions &LOpts) {
   if (LOpts.FakeAddressSpaceMap) {
     // The fake address space map must have a distinct entry for each
     // language-specific address space.
     static const unsigned FakeAddrSpaceMap[] = {
         0,  // Default
         1,  // opencl_global
         3,  // opencl_local
         2,  // opencl_constant
         0,  // opencl_private
         4,  // opencl_generic
         5,  // opencl_global_device
         6,  // opencl_global_host
         7,  // cuda_device
         8,  // cuda_constant
         9,  // cuda_shared
         1,  // sycl_global
         5,  // sycl_global_device
         6,  // sycl_global_host
         3,  // sycl_local
         0,  // sycl_private
         10, // ptr32_sptr
         11, // ptr32_uptr
         12  // ptr64
     };
     return &FakeAddrSpaceMap;
   } else {
     return &T.getAddressSpaceMap();
   }
 }
 
 static bool isAddrSpaceMapManglingEnabled(const TargetInfo &TI,
                                           const LangOptions &LangOpts) {
   switch (LangOpts.getAddressSpaceMapMangling()) {
   case LangOptions::ASMM_Target:
     return TI.useAddressSpaceMapMangling();
   case LangOptions::ASMM_On:
     return true;
   case LangOptions::ASMM_Off:
     return false;
   }
   llvm_unreachable("getAddressSpaceMapMangling() doesn't cover anything.");
 }
 
 ASTContext::ASTContext(LangOptions &LOpts, SourceManager &SM,
                        IdentifierTable &idents, SelectorTable &sels,
                        Builtin::Context &builtins, TranslationUnitKind TUKind)
     : ConstantArrayTypes(this_()), FunctionProtoTypes(this_()),
       TemplateSpecializationTypes(this_()),
       DependentTemplateSpecializationTypes(this_()), AutoTypes(this_()),
       SubstTemplateTemplateParmPacks(this_()),
       CanonTemplateTemplateParms(this_()), SourceMgr(SM), LangOpts(LOpts),
       NoSanitizeL(new NoSanitizeList(LangOpts.NoSanitizeFiles, SM)),
       XRayFilter(new XRayFunctionFilter(LangOpts.XRayAlwaysInstrumentFiles,
                                         LangOpts.XRayNeverInstrumentFiles,
                                         LangOpts.XRayAttrListFiles, SM)),
       ProfList(new ProfileList(LangOpts.ProfileListFiles, SM)),
       PrintingPolicy(LOpts), Idents(idents), Selectors(sels),
       BuiltinInfo(builtins), TUKind(TUKind), DeclarationNames(*this),
       Comments(SM), CommentCommandTraits(BumpAlloc, LOpts.CommentOpts),
       CompCategories(this_()), LastSDM(nullptr, 0) {
   addTranslationUnitDecl();
 }
 
 void ASTContext::cleanup() {
   // Release the DenseMaps associated with DeclContext objects.
   // FIXME: Is this the ideal solution?
   ReleaseDeclContextMaps();
 
   // Call all of the deallocation functions on all of their targets.
   for (auto &Pair : Deallocations)
     (Pair.first)(Pair.second);
   Deallocations.clear();
 
   // ASTRecordLayout objects in ASTRecordLayouts must always be destroyed
   // because they can contain DenseMaps.
   for (llvm::DenseMap<const ObjCContainerDecl*,
        const ASTRecordLayout*>::iterator
        I = ObjCLayouts.begin(), E = ObjCLayouts.end(); I != E; )
     // Increment in loop to prevent using deallocated memory.
     if (auto *R = const_cast<ASTRecordLayout *>((I++)->second))
       R->Destroy(*this);
   ObjCLayouts.clear();
 
   for (llvm::DenseMap<const RecordDecl*, const ASTRecordLayout*>::iterator
        I = ASTRecordLayouts.begin(), E = ASTRecordLayouts.end(); I != E; ) {
     // Increment in loop to prevent using deallocated memory.
     if (auto *R = const_cast<ASTRecordLayout *>((I++)->second))
       R->Destroy(*this);
   }
   ASTRecordLayouts.clear();
 
   for (llvm::DenseMap<const Decl*, AttrVec*>::iterator A = DeclAttrs.begin(),
                                                     AEnd = DeclAttrs.end();
        A != AEnd; ++A)
     A->second->~AttrVec();
   DeclAttrs.clear();
 
   for (const auto &Value : ModuleInitializers)
     Value.second->~PerModuleInitializers();
   ModuleInitializers.clear();
 }
 
 ASTContext::~ASTContext() { cleanup(); }
 
 void ASTContext::setTraversalScope(const std::vector<Decl *> &TopLevelDecls) {
   TraversalScope = TopLevelDecls;
   getParentMapContext().clear();
 }
 
 void ASTContext::AddDeallocation(void (*Callback)(void *), void *Data) const {
   Deallocations.push_back({Callback, Data});
 }
 
 void
 ASTContext::setExternalSource(IntrusiveRefCntPtr<ExternalASTSource> Source) {
   ExternalSource = std::move(Source);
 }
 
 void ASTContext::PrintStats() const {
   llvm::errs() << "\n*** AST Context Stats:\n";
   llvm::errs() << "  " << Types.size() << " types total.\n";
 
   unsigned counts[] = {
 #define TYPE(Name, Parent) 0,
 #define ABSTRACT_TYPE(Name, Parent)
 #include "clang/AST/TypeNodes.inc"
     0 // Extra
   };
 
   for (unsigned i = 0, e = Types.size(); i != e; ++i) {
     Type *T = Types[i];
     counts[(unsigned)T->getTypeClass()]++;
   }
 
   unsigned Idx = 0;
   unsigned TotalBytes = 0;
 #define TYPE(Name, Parent)                                              \
   if (counts[Idx])                                                      \
     llvm::errs() << "    " << counts[Idx] << " " << #Name               \
                  << " types, " << sizeof(Name##Type) << " each "        \
                  << "(" << counts[Idx] * sizeof(Name##Type)             \
                  << " bytes)\n";                                        \
   TotalBytes += counts[Idx] * sizeof(Name##Type);                       \
   ++Idx;
 #define ABSTRACT_TYPE(Name, Parent)
 #include "clang/AST/TypeNodes.inc"
 
   llvm::errs() << "Total bytes = " << TotalBytes << "\n";
 
   // Implicit special member functions.
   llvm::errs() << NumImplicitDefaultConstructorsDeclared << "/"
                << NumImplicitDefaultConstructors
                << " implicit default constructors created\n";
   llvm::errs() << NumImplicitCopyConstructorsDeclared << "/"
                << NumImplicitCopyConstructors
                << " implicit copy constructors created\n";
   if (getLangOpts().CPlusPlus)
     llvm::errs() << NumImplicitMoveConstructorsDeclared << "/"
                  << NumImplicitMoveConstructors
                  << " implicit move constructors created\n";
   llvm::errs() << NumImplicitCopyAssignmentOperatorsDeclared << "/"
                << NumImplicitCopyAssignmentOperators
                << " implicit copy assignment operators created\n";
   if (getLangOpts().CPlusPlus)
     llvm::errs() << NumImplicitMoveAssignmentOperatorsDeclared << "/"
                  << NumImplicitMoveAssignmentOperators
                  << " implicit move assignment operators created\n";
   llvm::errs() << NumImplicitDestructorsDeclared << "/"
                << NumImplicitDestructors
                << " implicit destructors created\n";
 
   if (ExternalSource) {
     llvm::errs() << "\n";
     ExternalSource->PrintStats();
   }
 
   BumpAlloc.PrintStats();
 }
 
 void ASTContext::mergeDefinitionIntoModule(NamedDecl *ND, Module *M,
                                            bool NotifyListeners) {
   if (NotifyListeners)
     if (auto *Listener = getASTMutationListener())
       Listener->RedefinedHiddenDefinition(ND, M);
 
   MergedDefModules[cast<NamedDecl>(ND->getCanonicalDecl())].push_back(M);
 }
 
 void ASTContext::deduplicateMergedDefinitonsFor(NamedDecl *ND) {
   auto It = MergedDefModules.find(cast<NamedDecl>(ND->getCanonicalDecl()));
   if (It == MergedDefModules.end())
     return;
 
   auto &Merged = It->second;
   llvm::DenseSet<Module*> Found;
   for (Module *&M : Merged)
     if (!Found.insert(M).second)
       M = nullptr;
   llvm::erase_value(Merged, nullptr);
 }
 
 ArrayRef<Module *>
 ASTContext::getModulesWithMergedDefinition(const NamedDecl *Def) {
   auto MergedIt =
       MergedDefModules.find(cast<NamedDecl>(Def->getCanonicalDecl()));
   if (MergedIt == MergedDefModules.end())
     return None;
   return MergedIt->second;
 }
 
 void ASTContext::PerModuleInitializers::resolve(ASTContext &Ctx) {
   if (LazyInitializers.empty())
     return;
 
   auto *Source = Ctx.getExternalSource();
   assert(Source && "lazy initializers but no external source");
 
   auto LazyInits = std::move(LazyInitializers);
   LazyInitializers.clear();
 
   for (auto ID : LazyInits)
     Initializers.push_back(Source->GetExternalDecl(ID));
 
   assert(LazyInitializers.empty() &&
          "GetExternalDecl for lazy module initializer added more inits");
 }
 
 void ASTContext::addModuleInitializer(Module *M, Decl *D) {
   // One special case: if we add a module initializer that imports another
   // module, and that module's only initializer is an ImportDecl, simplify.
   if (const auto *ID = dyn_cast<ImportDecl>(D)) {
     auto It = ModuleInitializers.find(ID->getImportedModule());
 
     // Maybe the ImportDecl does nothing at all. (Common case.)
     if (It == ModuleInitializers.end())
       return;
 
     // Maybe the ImportDecl only imports another ImportDecl.
     auto &Imported = *It->second;
     if (Imported.Initializers.size() + Imported.LazyInitializers.size() == 1) {
       Imported.resolve(*this);
       auto *OnlyDecl = Imported.Initializers.front();
       if (isa<ImportDecl>(OnlyDecl))
         D = OnlyDecl;
     }
   }
 
   auto *&Inits = ModuleInitializers[M];
   if (!Inits)
     Inits = new (*this) PerModuleInitializers;
   Inits->Initializers.push_back(D);
 }
 
 void ASTContext::addLazyModuleInitializers(Module *M, ArrayRef<uint32_t> IDs) {
   auto *&Inits = ModuleInitializers[M];
   if (!Inits)
     Inits = new (*this) PerModuleInitializers;
   Inits->LazyInitializers.insert(Inits->LazyInitializers.end(),
                                  IDs.begin(), IDs.end());
 }
 
 ArrayRef<Decl *> ASTContext::getModuleInitializers(Module *M) {
   auto It = ModuleInitializers.find(M);
   if (It == ModuleInitializers.end())
     return None;
 
   auto *Inits = It->second;
   Inits->resolve(*this);
   return Inits->Initializers;
 }
 
 ExternCContextDecl *ASTContext::getExternCContextDecl() const {
   if (!ExternCContext)
     ExternCContext = ExternCContextDecl::Create(*this, getTranslationUnitDecl());
 
   return ExternCContext;
 }
 
 BuiltinTemplateDecl *
 ASTContext::buildBuiltinTemplateDecl(BuiltinTemplateKind BTK,
                                      const IdentifierInfo *II) const {
   auto *BuiltinTemplate =
       BuiltinTemplateDecl::Create(*this, getTranslationUnitDecl(), II, BTK);
   BuiltinTemplate->setImplicit();
   getTranslationUnitDecl()->addDecl(BuiltinTemplate);
 
   return BuiltinTemplate;
 }
 
 BuiltinTemplateDecl *
 ASTContext::getMakeIntegerSeqDecl() const {
   if (!MakeIntegerSeqDecl)
     MakeIntegerSeqDecl = buildBuiltinTemplateDecl(BTK__make_integer_seq,
                                                   getMakeIntegerSeqName());
   return MakeIntegerSeqDecl;
 }
 
 BuiltinTemplateDecl *
 ASTContext::getTypePackElementDecl() const {
   if (!TypePackElementDecl)
     TypePackElementDecl = buildBuiltinTemplateDecl(BTK__type_pack_element,
                                                    getTypePackElementName());
   return TypePackElementDecl;
 }
 
 RecordDecl *ASTContext::buildImplicitRecord(StringRef Name,
                                             RecordDecl::TagKind TK) const {
   SourceLocation Loc;
   RecordDecl *NewDecl;
   if (getLangOpts().CPlusPlus)
     NewDecl = CXXRecordDecl::Create(*this, TK, getTranslationUnitDecl(), Loc,
                                     Loc, &Idents.get(Name));
   else
     NewDecl = RecordDecl::Create(*this, TK, getTranslationUnitDecl(), Loc, Loc,
                                  &Idents.get(Name));
   NewDecl->setImplicit();
   NewDecl->addAttr(TypeVisibilityAttr::CreateImplicit(
       const_cast<ASTContext &>(*this), TypeVisibilityAttr::Default));
   return NewDecl;
 }
 
 TypedefDecl *ASTContext::buildImplicitTypedef(QualType T,
                                               StringRef Name) const {
   TypeSourceInfo *TInfo = getTrivialTypeSourceInfo(T);
   TypedefDecl *NewDecl = TypedefDecl::Create(
       const_cast<ASTContext &>(*this), getTranslationUnitDecl(),
       SourceLocation(), SourceLocation(), &Idents.get(Name), TInfo);
   NewDecl->setImplicit();
   return NewDecl;
 }
 
 TypedefDecl *ASTContext::getInt128Decl() const {
   if (!Int128Decl)
     Int128Decl = buildImplicitTypedef(Int128Ty, "__int128_t");
   return Int128Decl;
 }
 
 TypedefDecl *ASTContext::getUInt128Decl() const {
   if (!UInt128Decl)
     UInt128Decl = buildImplicitTypedef(UnsignedInt128Ty, "__uint128_t");
   return UInt128Decl;
 }
 
 void ASTContext::InitBuiltinType(CanQualType &R, BuiltinType::Kind K) {
   auto *Ty = new (*this, TypeAlignment) BuiltinType(K);
   R = CanQualType::CreateUnsafe(QualType(Ty, 0));
   Types.push_back(Ty);
 }
 
 void ASTContext::InitBuiltinTypes(const TargetInfo &Target,
                                   const TargetInfo *AuxTarget) {
   assert((!this->Target || this->Target == &Target) &&
          "Incorrect target reinitialization");
   assert(VoidTy.isNull() && "Context reinitialized?");
 
   this->Target = &Target;
   this->AuxTarget = AuxTarget;
 
   ABI.reset(createCXXABI(Target));
   AddrSpaceMap = getAddressSpaceMap(Target, LangOpts);
   AddrSpaceMapMangling = isAddrSpaceMapManglingEnabled(Target, LangOpts);
 
   // C99 6.2.5p19.
   InitBuiltinType(VoidTy,              BuiltinType::Void);
 
   // C99 6.2.5p2.
   InitBuiltinType(BoolTy,              BuiltinType::Bool);
   // C99 6.2.5p3.
   if (LangOpts.CharIsSigned)
     InitBuiltinType(CharTy,            BuiltinType::Char_S);
   else
     InitBuiltinType(CharTy,            BuiltinType::Char_U);
   // C99 6.2.5p4.
   InitBuiltinType(SignedCharTy,        BuiltinType::SChar);
   InitBuiltinType(ShortTy,             BuiltinType::Short);
   InitBuiltinType(IntTy,               BuiltinType::Int);
   InitBuiltinType(LongTy,              BuiltinType::Long);
   InitBuiltinType(LongLongTy,          BuiltinType::LongLong);
 
   // C99 6.2.5p6.
   InitBuiltinType(UnsignedCharTy,      BuiltinType::UChar);
   InitBuiltinType(UnsignedShortTy,     BuiltinType::UShort);
   InitBuiltinType(UnsignedIntTy,       BuiltinType::UInt);
   InitBuiltinType(UnsignedLongTy,      BuiltinType::ULong);
   InitBuiltinType(UnsignedLongLongTy,  BuiltinType::ULongLong);
 
   // C99 6.2.5p10.
   InitBuiltinType(FloatTy,             BuiltinType::Float);
   InitBuiltinType(DoubleTy,            BuiltinType::Double);
   InitBuiltinType(LongDoubleTy,        BuiltinType::LongDouble);
 
   // GNU extension, __float128 for IEEE quadruple precision
   InitBuiltinType(Float128Ty,          BuiltinType::Float128);
 
   // __ibm128 for IBM extended precision
   InitBuiltinType(Ibm128Ty, BuiltinType::Ibm128);
 
   // C11 extension ISO/IEC TS 18661-3
   InitBuiltinType(Float16Ty,           BuiltinType::Float16);
 
   // ISO/IEC JTC1 SC22 WG14 N1169 Extension
   InitBuiltinType(ShortAccumTy,            BuiltinType::ShortAccum);
   InitBuiltinType(AccumTy,                 BuiltinType::Accum);
   InitBuiltinType(LongAccumTy,             BuiltinType::LongAccum);
   InitBuiltinType(UnsignedShortAccumTy,    BuiltinType::UShortAccum);
   InitBuiltinType(UnsignedAccumTy,         BuiltinType::UAccum);
   InitBuiltinType(UnsignedLongAccumTy,     BuiltinType::ULongAccum);
   InitBuiltinType(ShortFractTy,            BuiltinType::ShortFract);
   InitBuiltinType(FractTy,                 BuiltinType::Fract);
   InitBuiltinType(LongFractTy,             BuiltinType::LongFract);
   InitBuiltinType(UnsignedShortFractTy,    BuiltinType::UShortFract);
   InitBuiltinType(UnsignedFractTy,         BuiltinType::UFract);
   InitBuiltinType(UnsignedLongFractTy,     BuiltinType::ULongFract);
   InitBuiltinType(SatShortAccumTy,         BuiltinType::SatShortAccum);
   InitBuiltinType(SatAccumTy,              BuiltinType::SatAccum);
   InitBuiltinType(SatLongAccumTy,          BuiltinType::SatLongAccum);
   InitBuiltinType(SatUnsignedShortAccumTy, BuiltinType::SatUShortAccum);
   InitBuiltinType(SatUnsignedAccumTy,      BuiltinType::SatUAccum);
   InitBuiltinType(SatUnsignedLongAccumTy,  BuiltinType::SatULongAccum);
   InitBuiltinType(SatShortFractTy,         BuiltinType::SatShortFract);
   InitBuiltinType(SatFractTy,              BuiltinType::SatFract);
   InitBuiltinType(SatLongFractTy,          BuiltinType::SatLongFract);
   InitBuiltinType(SatUnsignedShortFractTy, BuiltinType::SatUShortFract);
   InitBuiltinType(SatUnsignedFractTy,      BuiltinType::SatUFract);
   InitBuiltinType(SatUnsignedLongFractTy,  BuiltinType::SatULongFract);
 
   // GNU extension, 128-bit integers.
   InitBuiltinType(Int128Ty,            BuiltinType::Int128);
   InitBuiltinType(UnsignedInt128Ty,    BuiltinType::UInt128);
 
   // C++ 3.9.1p5
   if (TargetInfo::isTypeSigned(Target.getWCharType()))
     InitBuiltinType(WCharTy,           BuiltinType::WChar_S);
   else  // -fshort-wchar makes wchar_t be unsigned.
     InitBuiltinType(WCharTy,           BuiltinType::WChar_U);
   if (LangOpts.CPlusPlus && LangOpts.WChar)
     WideCharTy = WCharTy;
   else {
     // C99 (or C++ using -fno-wchar).
     WideCharTy = getFromTargetType(Target.getWCharType());
   }
 
   WIntTy = getFromTargetType(Target.getWIntType());
 
   // C++20 (proposed)
   InitBuiltinType(Char8Ty,              BuiltinType::Char8);
 
   if (LangOpts.CPlusPlus) // C++0x 3.9.1p5, extension for C++
     InitBuiltinType(Char16Ty,           BuiltinType::Char16);
   else // C99
     Char16Ty = getFromTargetType(Target.getChar16Type());
 
   if (LangOpts.CPlusPlus) // C++0x 3.9.1p5, extension for C++
     InitBuiltinType(Char32Ty,           BuiltinType::Char32);
   else // C99
     Char32Ty = getFromTargetType(Target.getChar32Type());
 
   // Placeholder type for type-dependent expressions whose type is
   // completely unknown. No code should ever check a type against
   // DependentTy and users should never see it; however, it is here to
   // help diagnose failures to properly check for type-dependent
   // expressions.
   InitBuiltinType(DependentTy,         BuiltinType::Dependent);
 
   // Placeholder type for functions.
   InitBuiltinType(OverloadTy,          BuiltinType::Overload);
 
   // Placeholder type for bound members.
   InitBuiltinType(BoundMemberTy,       BuiltinType::BoundMember);
 
   // Placeholder type for pseudo-objects.
   InitBuiltinType(PseudoObjectTy,      BuiltinType::PseudoObject);
 
   // "any" type; useful for debugger-like clients.
   InitBuiltinType(UnknownAnyTy,        BuiltinType::UnknownAny);
 
   // Placeholder type for unbridged ARC casts.
   InitBuiltinType(ARCUnbridgedCastTy,  BuiltinType::ARCUnbridgedCast);
 
   // Placeholder type for builtin functions.
   InitBuiltinType(BuiltinFnTy,  BuiltinType::BuiltinFn);
 
   // Placeholder type for OMP array sections.
   if (LangOpts.OpenMP) {
     InitBuiltinType(OMPArraySectionTy, BuiltinType::OMPArraySection);
     InitBuiltinType(OMPArrayShapingTy, BuiltinType::OMPArrayShaping);
     InitBuiltinType(OMPIteratorTy, BuiltinType::OMPIterator);
   }
   if (LangOpts.MatrixTypes)
     InitBuiltinType(IncompleteMatrixIdxTy, BuiltinType::IncompleteMatrixIdx);
 
   // Builtin types for 'id', 'Class', and 'SEL'.
   InitBuiltinType(ObjCBuiltinIdTy, BuiltinType::ObjCId);
   InitBuiltinType(ObjCBuiltinClassTy, BuiltinType::ObjCClass);
   InitBuiltinType(ObjCBuiltinSelTy, BuiltinType::ObjCSel);
 
   if (LangOpts.OpenCL) {
 #define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
     InitBuiltinType(SingletonId, BuiltinType::Id);
 #include "clang/Basic/OpenCLImageTypes.def"
 
     InitBuiltinType(OCLSamplerTy, BuiltinType::OCLSampler);
     InitBuiltinType(OCLEventTy, BuiltinType::OCLEvent);
     InitBuiltinType(OCLClkEventTy, BuiltinType::OCLClkEvent);
     InitBuiltinType(OCLQueueTy, BuiltinType::OCLQueue);
     InitBuiltinType(OCLReserveIDTy, BuiltinType::OCLReserveID);
 
 #define EXT_OPAQUE_TYPE(ExtType, Id, Ext) \
     InitBuiltinType(Id##Ty, BuiltinType::Id);
 #include "clang/Basic/OpenCLExtensionTypes.def"
   }
 
   if (Target.hasAArch64SVETypes()) {
 #define SVE_TYPE(Name, Id, SingletonId) \
     InitBuiltinType(SingletonId, BuiltinType::Id);
 #include "clang/Basic/AArch64SVEACLETypes.def"
   }
 
   if (Target.getTriple().isPPC64()) {
 #define PPC_VECTOR_MMA_TYPE(Name, Id, Size) \
       InitBuiltinType(Id##Ty, BuiltinType::Id);
 #include "clang/Basic/PPCTypes.def"
 #define PPC_VECTOR_VSX_TYPE(Name, Id, Size) \
     InitBuiltinType(Id##Ty, BuiltinType::Id);
 #include "clang/Basic/PPCTypes.def"
   }
 
   if (Target.hasRISCVVTypes()) {
 #define RVV_TYPE(Name, Id, SingletonId)                                        \
   InitBuiltinType(SingletonId, BuiltinType::Id);
 #include "clang/Basic/RISCVVTypes.def"
   }
 
   // Builtin type for __objc_yes and __objc_no
   ObjCBuiltinBoolTy = (Target.useSignedCharForObjCBool() ?
                        SignedCharTy : BoolTy);
 
   ObjCConstantStringType = QualType();
 
   ObjCSuperType = QualType();
 
   // void * type
   if (LangOpts.OpenCLGenericAddressSpace) {
     auto Q = VoidTy.getQualifiers();
     Q.setAddressSpace(LangAS::opencl_generic);
     VoidPtrTy = getPointerType(getCanonicalType(
         getQualifiedType(VoidTy.getUnqualifiedType(), Q)));
   } else {
     VoidPtrTy = getPointerType(VoidTy);
   }
 
   // nullptr type (C++0x 2.14.7)
   InitBuiltinType(NullPtrTy,           BuiltinType::NullPtr);
 
   // half type (OpenCL 6.1.1.1) / ARM NEON __fp16
   InitBuiltinType(HalfTy, BuiltinType::Half);
 
   InitBuiltinType(BFloat16Ty, BuiltinType::BFloat16);
 
   // Builtin type used to help define __builtin_va_list.
   VaListTagDecl = nullptr;
 
   // MSVC predeclares struct _GUID, and we need it to create MSGuidDecls.
   if (LangOpts.MicrosoftExt || LangOpts.Borland) {
     MSGuidTagDecl = buildImplicitRecord("_GUID");
     getTranslationUnitDecl()->addDecl(MSGuidTagDecl);
   }
 }
 
 DiagnosticsEngine &ASTContext::getDiagnostics() const {
   return SourceMgr.getDiagnostics();
 }
 
 AttrVec& ASTContext::getDeclAttrs(const Decl *D) {
   AttrVec *&Result = DeclAttrs[D];
   if (!Result) {
     void *Mem = Allocate(sizeof(AttrVec));
     Result = new (Mem) AttrVec;
   }
 
   return *Result;
 }
 
 /// Erase the attributes corresponding to the given declaration.
 void ASTContext::eraseDeclAttrs(const Decl *D) {
   llvm::DenseMap<const Decl*, AttrVec*>::iterator Pos = DeclAttrs.find(D);
   if (Pos != DeclAttrs.end()) {
     Pos->second->~AttrVec();
     DeclAttrs.erase(Pos);
   }
 }
 
 // FIXME: Remove ?
 MemberSpecializationInfo *
 ASTContext::getInstantiatedFromStaticDataMember(const VarDecl *Var) {
   assert(Var->isStaticDataMember() && "Not a static data member");
   return getTemplateOrSpecializationInfo(Var)
       .dyn_cast<MemberSpecializationInfo *>();
 }
 
 ASTContext::TemplateOrSpecializationInfo
 ASTContext::getTemplateOrSpecializationInfo(const VarDecl *Var) {
   llvm::DenseMap<const VarDecl *, TemplateOrSpecializationInfo>::iterator Pos =
       TemplateOrInstantiation.find(Var);
   if (Pos == TemplateOrInstantiation.end())
     return {};
 
   return Pos->second;
 }
 
 void
 ASTContext::setInstantiatedFromStaticDataMember(VarDecl *Inst, VarDecl *Tmpl,
                                                 TemplateSpecializationKind TSK,
                                           SourceLocation PointOfInstantiation) {
   assert(Inst->isStaticDataMember() && "Not a static data member");
   assert(Tmpl->isStaticDataMember() && "Not a static data member");
   setTemplateOrSpecializationInfo(Inst, new (*this) MemberSpecializationInfo(
                                             Tmpl, TSK, PointOfInstantiation));
 }
 
 void
 ASTContext::setTemplateOrSpecializationInfo(VarDecl *Inst,
                                             TemplateOrSpecializationInfo TSI) {
   assert(!TemplateOrInstantiation[Inst] &&
          "Already noted what the variable was instantiated from");
   TemplateOrInstantiation[Inst] = TSI;
 }
 
 NamedDecl *
 ASTContext::getInstantiatedFromUsingDecl(NamedDecl *UUD) {
   auto Pos = InstantiatedFromUsingDecl.find(UUD);
   if (Pos == InstantiatedFromUsingDecl.end())
     return nullptr;
 
   return Pos->second;
 }
 
 void
 ASTContext::setInstantiatedFromUsingDecl(NamedDecl *Inst, NamedDecl *Pattern) {
   assert((isa<UsingDecl>(Pattern) ||
           isa<UnresolvedUsingValueDecl>(Pattern) ||
           isa<UnresolvedUsingTypenameDecl>(Pattern)) &&
          "pattern decl is not a using decl");
   assert((isa<UsingDecl>(Inst) ||
           isa<UnresolvedUsingValueDecl>(Inst) ||
           isa<UnresolvedUsingTypenameDecl>(Inst)) &&
          "instantiation did not produce a using decl");
   assert(!InstantiatedFromUsingDecl[Inst] && "pattern already exists");
   InstantiatedFromUsingDecl[Inst] = Pattern;
 }
 
 UsingEnumDecl *
 ASTContext::getInstantiatedFromUsingEnumDecl(UsingEnumDecl *UUD) {
   auto Pos = InstantiatedFromUsingEnumDecl.find(UUD);
   if (Pos == InstantiatedFromUsingEnumDecl.end())
     return nullptr;
 
   return Pos->second;
 }
 
 void ASTContext::setInstantiatedFromUsingEnumDecl(UsingEnumDecl *Inst,
                                                   UsingEnumDecl *Pattern) {
   assert(!InstantiatedFromUsingEnumDecl[Inst] && "pattern already exists");
   InstantiatedFromUsingEnumDecl[Inst] = Pattern;
 }
 
 UsingShadowDecl *
 ASTContext::getInstantiatedFromUsingShadowDecl(UsingShadowDecl *Inst) {
   llvm::DenseMap<UsingShadowDecl*, UsingShadowDecl*>::const_iterator Pos
     = InstantiatedFromUsingShadowDecl.find(Inst);
   if (Pos == InstantiatedFromUsingShadowDecl.end())
     return nullptr;
 
   return Pos->second;
 }
 
 void
 ASTContext::setInstantiatedFromUsingShadowDecl(UsingShadowDecl *Inst,
                                                UsingShadowDecl *Pattern) {
   assert(!InstantiatedFromUsingShadowDecl[Inst] && "pattern already exists");
   InstantiatedFromUsingShadowDecl[Inst] = Pattern;
 }
 
 FieldDecl *ASTContext::getInstantiatedFromUnnamedFieldDecl(FieldDecl *Field) {
   llvm::DenseMap<FieldDecl *, FieldDecl *>::iterator Pos
     = InstantiatedFromUnnamedFieldDecl.find(Field);
   if (Pos == InstantiatedFromUnnamedFieldDecl.end())
     return nullptr;
 
   return Pos->second;
 }
 
 void ASTContext::setInstantiatedFromUnnamedFieldDecl(FieldDecl *Inst,
                                                      FieldDecl *Tmpl) {
   assert(!Inst->getDeclName() && "Instantiated field decl is not unnamed");
   assert(!Tmpl->getDeclName() && "Template field decl is not unnamed");
   assert(!InstantiatedFromUnnamedFieldDecl[Inst] &&
          "Already noted what unnamed field was instantiated from");
 
   InstantiatedFromUnnamedFieldDecl[Inst] = Tmpl;
 }
 
 ASTContext::overridden_cxx_method_iterator
 ASTContext::overridden_methods_begin(const CXXMethodDecl *Method) const {
   return overridden_methods(Method).begin();
 }
 
 ASTContext::overridden_cxx_method_iterator
 ASTContext::overridden_methods_end(const CXXMethodDecl *Method) const {
   return overridden_methods(Method).end();
 }
 
 unsigned
 ASTContext::overridden_methods_size(const CXXMethodDecl *Method) const {
   auto Range = overridden_methods(Method);
   return Range.end() - Range.begin();
 }
 
 ASTContext::overridden_method_range
 ASTContext::overridden_methods(const CXXMethodDecl *Method) const {
   llvm::DenseMap<const CXXMethodDecl *, CXXMethodVector>::const_iterator Pos =
       OverriddenMethods.find(Method->getCanonicalDecl());
   if (Pos == OverriddenMethods.end())
     return overridden_method_range(nullptr, nullptr);
   return overridden_method_range(Pos->second.begin(), Pos->second.end());
 }
 
 void ASTContext::addOverriddenMethod(const CXXMethodDecl *Method,
                                      const CXXMethodDecl *Overridden) {
   assert(Method->isCanonicalDecl() && Overridden->isCanonicalDecl());
   OverriddenMethods[Method].push_back(Overridden);
 }
 
 void ASTContext::getOverriddenMethods(
                       const NamedDecl *D,
                       SmallVectorImpl<const NamedDecl *> &Overridden) const {
   assert(D);
 
   if (const auto *CXXMethod = dyn_cast<CXXMethodDecl>(D)) {
     Overridden.append(overridden_methods_begin(CXXMethod),
                       overridden_methods_end(CXXMethod));
     return;
   }
 
   const auto *Method = dyn_cast<ObjCMethodDecl>(D);
   if (!Method)
     return;
 
   SmallVector<const ObjCMethodDecl *, 8> OverDecls;
   Method->getOverriddenMethods(OverDecls);
   Overridden.append(OverDecls.begin(), OverDecls.end());
 }
 
 void ASTContext::addedLocalImportDecl(ImportDecl *Import) {
   assert(!Import->getNextLocalImport() &&
          "Import declaration already in the chain");
   assert(!Import->isFromASTFile() && "Non-local import declaration");
   if (!FirstLocalImport) {
     FirstLocalImport = Import;
     LastLocalImport = Import;
     return;
   }
 
   LastLocalImport->setNextLocalImport(Import);
   LastLocalImport = Import;
 }
 
 //===----------------------------------------------------------------------===//
 //                         Type Sizing and Analysis
 //===----------------------------------------------------------------------===//
 
 /// getFloatTypeSemantics - Return the APFloat 'semantics' for the specified
 /// scalar floating point type.
 const llvm::fltSemantics &ASTContext::getFloatTypeSemantics(QualType T) const {
   switch (T->castAs<BuiltinType>()->getKind()) {
   default:
     llvm_unreachable("Not a floating point type!");
   case BuiltinType::BFloat16:
     return Target->getBFloat16Format();
   case BuiltinType::Float16:
   case BuiltinType::Half:
     return Target->getHalfFormat();
   case BuiltinType::Float:      return Target->getFloatFormat();
   case BuiltinType::Double:     return Target->getDoubleFormat();
   case BuiltinType::Ibm128:
     return Target->getIbm128Format();
   case BuiltinType::LongDouble:
     if (getLangOpts().OpenMP && getLangOpts().OpenMPIsDevice)
       return AuxTarget->getLongDoubleFormat();
     return Target->getLongDoubleFormat();
   case BuiltinType::Float128:
     if (getLangOpts().OpenMP && getLangOpts().OpenMPIsDevice)
       return AuxTarget->getFloat128Format();
     return Target->getFloat128Format();
   }
 }
 
 CharUnits ASTContext::getDeclAlign(const Decl *D, bool ForAlignof) const {
   unsigned Align = Target->getCharWidth();
 
   bool UseAlignAttrOnly = false;
   if (unsigned AlignFromAttr = D->getMaxAlignment()) {
     Align = AlignFromAttr;
 
     // __attribute__((aligned)) can increase or decrease alignment
     // *except* on a struct or struct member, where it only increases
     // alignment unless 'packed' is also specified.
     //
     // It is an error for alignas to decrease alignment, so we can
     // ignore that possibility;  Sema should diagnose it.
     if (isa<FieldDecl>(D)) {
       UseAlignAttrOnly = D->hasAttr<PackedAttr>() ||
         cast<FieldDecl>(D)->getParent()->hasAttr<PackedAttr>();
     } else {
       UseAlignAttrOnly = true;
     }
   }
   else if (isa<FieldDecl>(D))
       UseAlignAttrOnly =
         D->hasAttr<PackedAttr>() ||
         cast<FieldDecl>(D)->getParent()->hasAttr<PackedAttr>();
 
   // If we're using the align attribute only, just ignore everything
   // else about the declaration and its type.
   if (UseAlignAttrOnly) {
     // do nothing
   } else if (const auto *VD = dyn_cast<ValueDecl>(D)) {
     QualType T = VD->getType();
     if (const auto *RT = T->getAs<ReferenceType>()) {
       if (ForAlignof)
         T = RT->getPointeeType();
       else
         T = getPointerType(RT->getPointeeType());
     }
     QualType BaseT = getBaseElementType(T);
     if (T->isFunctionType())
       Align = getTypeInfoImpl(T.getTypePtr()).Align;
     else if (!BaseT->isIncompleteType()) {
       // Adjust alignments of declarations with array type by the
       // large-array alignment on the target.
       if (const ArrayType *arrayType = getAsArrayType(T)) {
         unsigned MinWidth = Target->getLargeArrayMinWidth();
         if (!ForAlignof && MinWidth) {
           if (isa<VariableArrayType>(arrayType))
             Align = std::max(Align, Target->getLargeArrayAlign());
           else if (isa<ConstantArrayType>(arrayType) &&
                    MinWidth <= getTypeSize(cast<ConstantArrayType>(arrayType)))
             Align = std::max(Align, Target->getLargeArrayAlign());
         }
       }
       Align = std::max(Align, getPreferredTypeAlign(T.getTypePtr()));
       if (BaseT.getQualifiers().hasUnaligned())
         Align = Target->getCharWidth();
       if (const auto *VD = dyn_cast<VarDecl>(D)) {
         if (VD->hasGlobalStorage() && !ForAlignof) {
           uint64_t TypeSize = getTypeSize(T.getTypePtr());
           Align = std::max(Align, getTargetInfo().getMinGlobalAlign(TypeSize));
         }
       }
     }
 
     // Fields can be subject to extra alignment constraints, like if
     // the field is packed, the struct is packed, or the struct has a
     // a max-field-alignment constraint (#pragma pack).  So calculate
     // the actual alignment of the field within the struct, and then
     // (as we're expected to) constrain that by the alignment of the type.
     if (const auto *Field = dyn_cast<FieldDecl>(VD)) {
       const RecordDecl *Parent = Field->getParent();
       // We can only produce a sensible answer if the record is valid.
       if (!Parent->isInvalidDecl()) {
         const ASTRecordLayout &Layout = getASTRecordLayout(Parent);
 
         // Start with the record's overall alignment.
         unsigned FieldAlign = toBits(Layout.getAlignment());
 
         // Use the GCD of that and the offset within the record.
         uint64_t Offset = Layout.getFieldOffset(Field->getFieldIndex());
         if (Offset > 0) {
           // Alignment is always a power of 2, so the GCD will be a power of 2,
           // which means we get to do this crazy thing instead of Euclid's.
           uint64_t LowBitOfOffset = Offset & (~Offset + 1);
           if (LowBitOfOffset < FieldAlign)
             FieldAlign = static_cast<unsigned>(LowBitOfOffset);
         }
 
         Align = std::min(Align, FieldAlign);
       }
     }
   }
 
   // Some targets have hard limitation on the maximum requestable alignment in
   // aligned attribute for static variables.
   const unsigned MaxAlignedAttr = getTargetInfo().getMaxAlignedAttribute();
   const auto *VD = dyn_cast<VarDecl>(D);
   if (MaxAlignedAttr && VD && VD->getStorageClass() == SC_Static)
     Align = std::min(Align, MaxAlignedAttr);
 
   return toCharUnitsFromBits(Align);
 }
 
 CharUnits ASTContext::getExnObjectAlignment() const {
   return toCharUnitsFromBits(Target->getExnObjectAlignment());
 }
 
 // getTypeInfoDataSizeInChars - Return the size of a type, in
 // chars. If the type is a record, its data size is returned.  This is
 // the size of the memcpy that's performed when assigning this type
 // using a trivial copy/move assignment operator.
 TypeInfoChars ASTContext::getTypeInfoDataSizeInChars(QualType T) const {
   TypeInfoChars Info = getTypeInfoInChars(T);
 
   // In C++, objects can sometimes be allocated into the tail padding
   // of a base-class subobject.  We decide whether that's possible
   // during class layout, so here we can just trust the layout results.
   if (getLangOpts().CPlusPlus) {
     if (const auto *RT = T->getAs<RecordType>()) {
       const ASTRecordLayout &layout = getASTRecordLayout(RT->getDecl());
       Info.Width = layout.getDataSize();
     }
   }
 
   return Info;
 }
 
 /// getConstantArrayInfoInChars - Performing the computation in CharUnits
 /// instead of in bits prevents overflowing the uint64_t for some large arrays.
 TypeInfoChars
 static getConstantArrayInfoInChars(const ASTContext &Context,
                                    const ConstantArrayType *CAT) {
   TypeInfoChars EltInfo = Context.getTypeInfoInChars(CAT->getElementType());
   uint64_t Size = CAT->getSize().getZExtValue();
   assert((Size == 0 || static_cast<uint64_t>(EltInfo.Width.getQuantity()) <=
               (uint64_t)(-1)/Size) &&
          "Overflow in array type char size evaluation");
   uint64_t Width = EltInfo.Width.getQuantity() * Size;
   unsigned Align = EltInfo.Align.getQuantity();
   if (!Context.getTargetInfo().getCXXABI().isMicrosoft() ||
       Context.getTargetInfo().getPointerWidth(0) == 64)
     Width = llvm::alignTo(Width, Align);
   return TypeInfoChars(CharUnits::fromQuantity(Width),
                        CharUnits::fromQuantity(Align),
                        EltInfo.AlignRequirement);
 }
 
 TypeInfoChars ASTContext::getTypeInfoInChars(const Type *T) const {
   if (const auto *CAT = dyn_cast<ConstantArrayType>(T))
     return getConstantArrayInfoInChars(*this, CAT);
   TypeInfo Info = getTypeInfo(T);
   return TypeInfoChars(toCharUnitsFromBits(Info.Width),
                        toCharUnitsFromBits(Info.Align), Info.AlignRequirement);
 }
 
 TypeInfoChars ASTContext::getTypeInfoInChars(QualType T) const {
   return getTypeInfoInChars(T.getTypePtr());
 }
 
 bool ASTContext::isAlignmentRequired(const Type *T) const {
   return getTypeInfo(T).AlignRequirement != AlignRequirementKind::None;
 }
 
 bool ASTContext::isAlignmentRequired(QualType T) const {
   return isAlignmentRequired(T.getTypePtr());
 }
 
 unsigned ASTContext::getTypeAlignIfKnown(QualType T,
                                          bool NeedsPreferredAlignment) const {
   // An alignment on a typedef overrides anything else.
   if (const auto *TT = T->getAs<TypedefType>())
     if (unsigned Align = TT->getDecl()->getMaxAlignment())
       return Align;
 
   // If we have an (array of) complete type, we're done.
   T = getBaseElementType(T);
   if (!T->isIncompleteType())
     return NeedsPreferredAlignment ? getPreferredTypeAlign(T) : getTypeAlign(T);
 
   // If we had an array type, its element type might be a typedef
   // type with an alignment attribute.
   if (const auto *TT = T->getAs<TypedefType>())
     if (unsigned Align = TT->getDecl()->getMaxAlignment())
       return Align;
 
   // Otherwise, see if the declaration of the type had an attribute.
   if (const auto *TT = T->getAs<TagType>())
     return TT->getDecl()->getMaxAlignment();
 
   return 0;
 }
 
 TypeInfo ASTContext::getTypeInfo(const Type *T) const {
   TypeInfoMap::iterator I = MemoizedTypeInfo.find(T);
   if (I != MemoizedTypeInfo.end())
     return I->second;
 
   // This call can invalidate MemoizedTypeInfo[T], so we need a second lookup.
   TypeInfo TI = getTypeInfoImpl(T);
   MemoizedTypeInfo[T] = TI;
   return TI;
 }
 
 /// getTypeInfoImpl - Return the size of the specified type, in bits.  This
 /// method does not work on incomplete types.
 ///
 /// FIXME: Pointers into different addr spaces could have different sizes and
 /// alignment requirements: getPointerInfo should take an AddrSpace, this
 /// should take a QualType, &c.
 TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
   uint64_t Width = 0;
   unsigned Align = 8;
   AlignRequirementKind AlignRequirement = AlignRequirementKind::None;
   unsigned AS = 0;
   switch (T->getTypeClass()) {
 #define TYPE(Class, Base)
 #define ABSTRACT_TYPE(Class, Base)
 #define NON_CANONICAL_TYPE(Class, Base)
 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)                       \
   case Type::Class:                                                            \
   assert(!T->isDependentType() && "should not see dependent types here");      \
   return getTypeInfo(cast<Class##Type>(T)->desugar().getTypePtr());
 #include "clang/AST/TypeNodes.inc"
     llvm_unreachable("Should not see dependent types");
 
   case Type::FunctionNoProto:
   case Type::FunctionProto:
     // GCC extension: alignof(function) = 32 bits
     Width = 0;
     Align = 32;
     break;
 
   case Type::IncompleteArray:
   case Type::VariableArray:
   case Type::ConstantArray: {
     // Model non-constant sized arrays as size zero, but track the alignment.
     uint64_t Size = 0;
     if (const auto *CAT = dyn_cast<ConstantArrayType>(T))
       Size = CAT->getSize().getZExtValue();
 
     TypeInfo EltInfo = getTypeInfo(cast<ArrayType>(T)->getElementType());
     assert((Size == 0 || EltInfo.Width <= (uint64_t)(-1) / Size) &&
            "Overflow in array type bit size evaluation");
     Width = EltInfo.Width * Size;
     Align = EltInfo.Align;
     AlignRequirement = EltInfo.AlignRequirement;
     if (!getTargetInfo().getCXXABI().isMicrosoft() ||
         getTargetInfo().getPointerWidth(0) == 64)
       Width = llvm::alignTo(Width, Align);
     break;
   }
 
   case Type::ExtVector:
   case Type::Vector: {
     const auto *VT = cast<VectorType>(T);
     TypeInfo EltInfo = getTypeInfo(VT->getElementType());
     Width = EltInfo.Width * VT->getNumElements();
     Align = Width;
     // If the alignment is not a power of 2, round up to the next power of 2.
     // This happens for non-power-of-2 length vectors.
     if (Align & (Align-1)) {
       Align = llvm::NextPowerOf2(Align);
       Width = llvm::alignTo(Width, Align);
     }
     // Adjust the alignment based on the target max.
     uint64_t TargetVectorAlign = Target->getMaxVectorAlign();
     if (TargetVectorAlign && TargetVectorAlign < Align)
       Align = TargetVectorAlign;
     if (VT->getVectorKind() == VectorType::SveFixedLengthDataVector)
       // Adjust the alignment for fixed-length SVE vectors. This is important
       // for non-power-of-2 vector lengths.
       Align = 128;
     else if (VT->getVectorKind() == VectorType::SveFixedLengthPredicateVector)
       // Adjust the alignment for fixed-length SVE predicates.
       Align = 16;
     break;
   }
 
   case Type::ConstantMatrix: {
     const auto *MT = cast<ConstantMatrixType>(T);
     TypeInfo ElementInfo = getTypeInfo(MT->getElementType());
     // The internal layout of a matrix value is implementation defined.
     // Initially be ABI compatible with arrays with respect to alignment and
     // size.
     Width = ElementInfo.Width * MT->getNumRows() * MT->getNumColumns();
     Align = ElementInfo.Align;
     break;
   }
 
   case Type::Builtin:
     switch (cast<BuiltinType>(T)->getKind()) {
     default: llvm_unreachable("Unknown builtin type!");
     case BuiltinType::Void:
       // GCC extension: alignof(void) = 8 bits.
       Width = 0;
       Align = 8;
       break;
     case BuiltinType::Bool:
       Width = Target->getBoolWidth();
       Align = Target->getBoolAlign();
       break;
     case BuiltinType::Char_S:
     case BuiltinType::Char_U:
     case BuiltinType::UChar:
     case BuiltinType::SChar:
     case BuiltinType::Char8:
       Width = Target->getCharWidth();
       Align = Target->getCharAlign();
       break;
     case BuiltinType::WChar_S:
     case BuiltinType::WChar_U:
       Width = Target->getWCharWidth();
       Align = Target->getWCharAlign();
       break;
     case BuiltinType::Char16:
       Width = Target->getChar16Width();
       Align = Target->getChar16Align();
       break;
     case BuiltinType::Char32:
       Width = Target->getChar32Width();
       Align = Target->getChar32Align();
       break;
     case BuiltinType::UShort:
     case BuiltinType::Short:
       Width = Target->getShortWidth();
       Align = Target->getShortAlign();
       break;
     case BuiltinType::UInt:
     case BuiltinType::Int:
       Width = Target->getIntWidth();
       Align = Target->getIntAlign();
       break;
     case BuiltinType::ULong:
     case BuiltinType::Long:
       Width = Target->getLongWidth();
       Align = Target->getLongAlign();
       break;
     case BuiltinType::ULongLong:
     case BuiltinType::LongLong:
       Width = Target->getLongLongWidth();
       Align = Target->getLongLongAlign();
       break;
     case BuiltinType::Int128:
     case BuiltinType::UInt128:
       Width = 128;
       Align = 128; // int128_t is 128-bit aligned on all targets.
       break;
     case BuiltinType::ShortAccum:
     case BuiltinType::UShortAccum:
     case BuiltinType::SatShortAccum:
     case BuiltinType::SatUShortAccum:
       Width = Target->getShortAccumWidth();
       Align = Target->getShortAccumAlign();
       break;
     case BuiltinType::Accum:
     case BuiltinType::UAccum:
     case BuiltinType::SatAccum:
     case BuiltinType::SatUAccum:
       Width = Target->getAccumWidth();
       Align = Target->getAccumAlign();
       break;
     case BuiltinType::LongAccum:
     case BuiltinType::ULongAccum:
     case BuiltinType::SatLongAccum:
     case BuiltinType::SatULongAccum:
       Width = Target->getLongAccumWidth();
       Align = Target->getLongAccumAlign();
       break;
     case BuiltinType::ShortFract:
     case BuiltinType::UShortFract:
     case BuiltinType::SatShortFract:
     case BuiltinType::SatUShortFract:
       Width = Target->getShortFractWidth();
       Align = Target->getShortFractAlign();
       break;
     case BuiltinType::Fract:
     case BuiltinType::UFract:
     case BuiltinType::SatFract:
     case BuiltinType::SatUFract:
       Width = Target->getFractWidth();
       Align = Target->getFractAlign();
       break;
     case BuiltinType::LongFract:
     case BuiltinType::ULongFract:
     case BuiltinType::SatLongFract:
     case BuiltinType::SatULongFract:
       Width = Target->getLongFractWidth();
       Align = Target->getLongFractAlign();
       break;
     case BuiltinType::BFloat16:
       Width = Target->getBFloat16Width();
       Align = Target->getBFloat16Align();
       break;
     case BuiltinType::Float16:
     case BuiltinType::Half:
       if (Target->hasFloat16Type() || !getLangOpts().OpenMP ||
           !getLangOpts().OpenMPIsDevice) {
         Width = Target->getHalfWidth();
         Align = Target->getHalfAlign();
       } else {
         assert(getLangOpts().OpenMP && getLangOpts().OpenMPIsDevice &&
                "Expected OpenMP device compilation.");
         Width = AuxTarget->getHalfWidth();
         Align = AuxTarget->getHalfAlign();
       }
       break;
     case BuiltinType::Float:
       Width = Target->getFloatWidth();
       Align = Target->getFloatAlign();
       break;
     case BuiltinType::Double:
       Width = Target->getDoubleWidth();
       Align = Target->getDoubleAlign();
       break;
     case BuiltinType::Ibm128:
       Width = Target->getIbm128Width();
       Align = Target->getIbm128Align();
       break;
     case BuiltinType::LongDouble:
       if (getLangOpts().OpenMP && getLangOpts().OpenMPIsDevice &&
           (Target->getLongDoubleWidth() != AuxTarget->getLongDoubleWidth() ||
            Target->getLongDoubleAlign() != AuxTarget->getLongDoubleAlign())) {
         Width = AuxTarget->getLongDoubleWidth();
         Align = AuxTarget->getLongDoubleAlign();
       } else {
         Width = Target->getLongDoubleWidth();
         Align = Target->getLongDoubleAlign();
       }
       break;
     case BuiltinType::Float128:
       if (Target->hasFloat128Type() || !getLangOpts().OpenMP ||
           !getLangOpts().OpenMPIsDevice) {
         Width = Target->getFloat128Width();
         Align = Target->getFloat128Align();
       } else {
         assert(getLangOpts().OpenMP && getLangOpts().OpenMPIsDevice &&
                "Expected OpenMP device compilation.");
         Width = AuxTarget->getFloat128Width();
         Align = AuxTarget->getFloat128Align();
       }
       break;
     case BuiltinType::NullPtr:
       Width = Target->getPointerWidth(0); // C++ 3.9.1p11: sizeof(nullptr_t)
       Align = Target->getPointerAlign(0); //   == sizeof(void*)
       break;
     case BuiltinType::ObjCId:
     case BuiltinType::ObjCClass:
     case BuiltinType::ObjCSel:
       Width = Target->getPointerWidth(0);
       Align = Target->getPointerAlign(0);
       break;
     case BuiltinType::OCLSampler:
     case BuiltinType::OCLEvent:
     case BuiltinType::OCLClkEvent:
     case BuiltinType::OCLQueue:
     case BuiltinType::OCLReserveID:
 #define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
     case BuiltinType::Id:
 #include "clang/Basic/OpenCLImageTypes.def"
 #define EXT_OPAQUE_TYPE(ExtType, Id, Ext) \
   case BuiltinType::Id:
 #include "clang/Basic/OpenCLExtensionTypes.def"
       AS = getTargetAddressSpace(
           Target->getOpenCLTypeAddrSpace(getOpenCLTypeKind(T)));
       Width = Target->getPointerWidth(AS);
       Align = Target->getPointerAlign(AS);
       break;
     // The SVE types are effectively target-specific.  The length of an
     // SVE_VECTOR_TYPE is only known at runtime, but it is always a multiple
     // of 128 bits.  There is one predicate bit for each vector byte, so the
     // length of an SVE_PREDICATE_TYPE is always a multiple of 16 bits.
     //
     // Because the length is only known at runtime, we use a dummy value
     // of 0 for the static length.  The alignment values are those defined
     // by the Procedure Call Standard for the Arm Architecture.
 #define SVE_VECTOR_TYPE(Name, MangledName, Id, SingletonId, NumEls, ElBits,    \
                         IsSigned, IsFP, IsBF)                                  \
   case BuiltinType::Id:                                                        \
     Width = 0;                                                                 \
     Align = 128;                                                               \
     break;
 #define SVE_PREDICATE_TYPE(Name, MangledName, Id, SingletonId, NumEls)         \
   case BuiltinType::Id:                                                        \
     Width = 0;                                                                 \
     Align = 16;                                                                \
     break;
 #include "clang/Basic/AArch64SVEACLETypes.def"
 #define PPC_VECTOR_TYPE(Name, Id, Size)                                        \
   case BuiltinType::Id:                                                        \
     Width = Size;                                                              \
     Align = Size;                                                              \
     break;
 #include "clang/Basic/PPCTypes.def"
 #define RVV_VECTOR_TYPE(Name, Id, SingletonId, ElKind, ElBits, NF, IsSigned,   \
                         IsFP)                                                  \
   case BuiltinType::Id:                                                        \
     Width = 0;                                                                 \
     Align = ElBits;                                                            \
     break;
 #define RVV_PREDICATE_TYPE(Name, Id, SingletonId, ElKind)                      \
   case BuiltinType::Id:                                                        \
     Width = 0;                                                                 \
     Align = 8;                                                                 \
     break;
 #include "clang/Basic/RISCVVTypes.def"
     }
     break;
   case Type::ObjCObjectPointer:
     Width = Target->getPointerWidth(0);
     Align = Target->getPointerAlign(0);
     break;
   case Type::BlockPointer:
     AS = getTargetAddressSpace(cast<BlockPointerType>(T)->getPointeeType());
     Width = Target->getPointerWidth(AS);
     Align = Target->getPointerAlign(AS);
     break;
   case Type::LValueReference:
   case Type::RValueReference:
     // alignof and sizeof should never enter this code path here, so we go
     // the pointer route.
     AS = getTargetAddressSpace(cast<ReferenceType>(T)->getPointeeType());
     Width = Target->getPointerWidth(AS);
     Align = Target->getPointerAlign(AS);
     break;
   case Type::Pointer:
     AS = getTargetAddressSpace(cast<PointerType>(T)->getPointeeType());
     Width = Target->getPointerWidth(AS);
     Align = Target->getPointerAlign(AS);
     break;
   case Type::MemberPointer: {
     const auto *MPT = cast<MemberPointerType>(T);
     CXXABI::MemberPointerInfo MPI = ABI->getMemberPointerInfo(MPT);
     Width = MPI.Width;
     Align = MPI.Align;
     break;
   }
   case Type::Complex: {
     // Complex types have the same alignment as their elements, but twice the
     // size.
     TypeInfo EltInfo = getTypeInfo(cast<ComplexType>(T)->getElementType());
     Width = EltInfo.Width * 2;
     Align = EltInfo.Align;
     break;
   }
   case Type::ObjCObject:
     return getTypeInfo(cast<ObjCObjectType>(T)->getBaseType().getTypePtr());
   case Type::Adjusted:
   case Type::Decayed:
     return getTypeInfo(cast<AdjustedType>(T)->getAdjustedType().getTypePtr());
   case Type::ObjCInterface: {
     const auto *ObjCI = cast<ObjCInterfaceType>(T);
     if (ObjCI->getDecl()->isInvalidDecl()) {
       Width = 8;
       Align = 8;
       break;
     }
     const ASTRecordLayout &Layout = getASTObjCInterfaceLayout(ObjCI->getDecl());
     Width = toBits(Layout.getSize());
     Align = toBits(Layout.getAlignment());
     break;
   }
   case Type::BitInt: {
     const auto *EIT = cast<BitIntType>(T);
     Align =
         std::min(static_cast<unsigned>(std::max(
                      getCharWidth(), llvm::PowerOf2Ceil(EIT->getNumBits()))),
                  Target->getLongLongAlign());
     Width = llvm::alignTo(EIT->getNumBits(), Align);
     break;
   }
   case Type::Record:
   case Type::Enum: {
     const auto *TT = cast<TagType>(T);
 
     if (TT->getDecl()->isInvalidDecl()) {
       Width = 8;
       Align = 8;
       break;
     }
 
     if (const auto *ET = dyn_cast<EnumType>(TT)) {
       const EnumDecl *ED = ET->getDecl();
       TypeInfo Info =
           getTypeInfo(ED->getIntegerType()->getUnqualifiedDesugaredType());
       if (unsigned AttrAlign = ED->getMaxAlignment()) {
         Info.Align = AttrAlign;
         Info.AlignRequirement = AlignRequirementKind::RequiredByEnum;
       }
       return Info;
     }
 
     const auto *RT = cast<RecordType>(TT);
     const RecordDecl *RD = RT->getDecl();
     const ASTRecordLayout &Layout = getASTRecordLayout(RD);
     Width = toBits(Layout.getSize());
     Align = toBits(Layout.getAlignment());
     AlignRequirement = RD->hasAttr<AlignedAttr>()
                            ? AlignRequirementKind::RequiredByRecord
                            : AlignRequirementKind::None;
     break;
   }
 
   case Type::SubstTemplateTypeParm:
     return getTypeInfo(cast<SubstTemplateTypeParmType>(T)->
                        getReplacementType().getTypePtr());
 
   case Type::Auto:
   case Type::DeducedTemplateSpecialization: {
     const auto *A = cast<DeducedType>(T);
     assert(!A->getDeducedType().isNull() &&
            "cannot request the size of an undeduced or dependent auto type");
     return getTypeInfo(A->getDeducedType().getTypePtr());
   }
 
   case Type::Paren:
     return getTypeInfo(cast<ParenType>(T)->getInnerType().getTypePtr());
 
   case Type::MacroQualified:
     return getTypeInfo(
         cast<MacroQualifiedType>(T)->getUnderlyingType().getTypePtr());
 
   case Type::ObjCTypeParam:
     return getTypeInfo(cast<ObjCTypeParamType>(T)->desugar().getTypePtr());
 
   case Type::Using:
     return getTypeInfo(cast<UsingType>(T)->desugar().getTypePtr());
 
   case Type::Typedef: {
     const TypedefNameDecl *Typedef = cast<TypedefType>(T)->getDecl();
     TypeInfo Info = getTypeInfo(Typedef->getUnderlyingType().getTypePtr());
     // If the typedef has an aligned attribute on it, it overrides any computed
     // alignment we have.  This violates the GCC documentation (which says that
     // attribute(aligned) can only round up) but matches its implementation.
     if (unsigned AttrAlign = Typedef->getMaxAlignment()) {
       Align = AttrAlign;
       AlignRequirement = AlignRequirementKind::RequiredByTypedef;
     } else {
       Align = Info.Align;
       AlignRequirement = Info.AlignRequirement;
     }
     Width = Info.Width;
     break;
   }
 
   case Type::Elaborated:
     return getTypeInfo(cast<ElaboratedType>(T)->getNamedType().getTypePtr());
 
   case Type::Attributed:
     return getTypeInfo(
                   cast<AttributedType>(T)->getEquivalentType().getTypePtr());
 
   case Type::Atomic: {
     // Start with the base type information.
     TypeInfo Info = getTypeInfo(cast<AtomicType>(T)->getValueType());
     Width = Info.Width;
     Align = Info.Align;
 
     if (!Width) {
       // An otherwise zero-sized type should still generate an
       // atomic operation.
       Width = Target->getCharWidth();
       assert(Align);
     } else if (Width <= Target->getMaxAtomicPromoteWidth()) {
       // If the size of the type doesn't exceed the platform's max
       // atomic promotion width, make the size and alignment more
       // favorable to atomic operations:
 
       // Round the size up to a power of 2.
       if (!llvm::isPowerOf2_64(Width))
         Width = llvm::NextPowerOf2(Width);
 
       // Set the alignment equal to the size.
       Align = static_cast<unsigned>(Width);
     }
   }
   break;
 
   case Type::Pipe:
     Width = Target->getPointerWidth(getTargetAddressSpace(LangAS::opencl_global));
     Align = Target->getPointerAlign(getTargetAddressSpace(LangAS::opencl_global));
     break;
   }
 
   assert(llvm::isPowerOf2_32(Align) && "Alignment must be power of 2");
   return TypeInfo(Width, Align, AlignRequirement);
 }
 
 unsigned ASTContext::getTypeUnadjustedAlign(const Type *T) const {
   UnadjustedAlignMap::iterator I = MemoizedUnadjustedAlign.find(T);
   if (I != MemoizedUnadjustedAlign.end())
     return I->second;
 
   unsigned UnadjustedAlign;
   if (const auto *RT = T->getAs<RecordType>()) {
     const RecordDecl *RD = RT->getDecl();
     const ASTRecordLayout &Layout = getASTRecordLayout(RD);
     UnadjustedAlign = toBits(Layout.getUnadjustedAlignment());
   } else if (const auto *ObjCI = T->getAs<ObjCInterfaceType>()) {
     const ASTRecordLayout &Layout = getASTObjCInterfaceLayout(ObjCI->getDecl());
     UnadjustedAlign = toBits(Layout.getUnadjustedAlignment());
   } else {
     UnadjustedAlign = getTypeAlign(T->getUnqualifiedDesugaredType());
   }
 
   MemoizedUnadjustedAlign[T] = UnadjustedAlign;
   return UnadjustedAlign;
 }
 
 unsigned ASTContext::getOpenMPDefaultSimdAlign(QualType T) const {
   unsigned SimdAlign = getTargetInfo().getSimdDefaultAlign();
   return SimdAlign;
 }
 
 /// toCharUnitsFromBits - Convert a size in bits to a size in characters.
 CharUnits ASTContext::toCharUnitsFromBits(int64_t BitSize) const {
   return CharUnits::fromQuantity(BitSize / getCharWidth());
 }
 
 /// toBits - Convert a size in characters to a size in characters.
 int64_t ASTContext::toBits(CharUnits CharSize) const {
   return CharSize.getQuantity() * getCharWidth();
 }
 
 /// getTypeSizeInChars - Return the size of the specified type, in characters.
 /// This method does not work on incomplete types.
 CharUnits ASTContext::getTypeSizeInChars(QualType T) const {
   return getTypeInfoInChars(T).Width;
 }
 CharUnits ASTContext::getTypeSizeInChars(const Type *T) const {
   return getTypeInfoInChars(T).Width;
 }
 
 /// getTypeAlignInChars - Return the ABI-specified alignment of a type, in
 /// characters. This method does not work on incomplete types.
 CharUnits ASTContext::getTypeAlignInChars(QualType T) const {
   return toCharUnitsFromBits(getTypeAlign(T));
 }
 CharUnits ASTContext::getTypeAlignInChars(const Type *T) const {
   return toCharUnitsFromBits(getTypeAlign(T));
 }
 
 /// getTypeUnadjustedAlignInChars - Return the ABI-specified alignment of a
 /// type, in characters, before alignment adustments. This method does
 /// not work on incomplete types.
 CharUnits ASTContext::getTypeUnadjustedAlignInChars(QualType T) const {
   return toCharUnitsFromBits(getTypeUnadjustedAlign(T));
 }
 CharUnits ASTContext::getTypeUnadjustedAlignInChars(const Type *T) const {
   return toCharUnitsFromBits(getTypeUnadjustedAlign(T));
 }
 
 /// getPreferredTypeAlign - Return the "preferred" alignment of the specified
 /// type for the current target in bits.  This can be different than the ABI
 /// alignment in cases where it is beneficial for performance or backwards
 /// compatibility preserving to overalign a data type. (Note: despite the name,
 /// the preferred alignment is ABI-impacting, and not an optimization.)
 unsigned ASTContext::getPreferredTypeAlign(const Type *T) const {
   TypeInfo TI = getTypeInfo(T);
   unsigned ABIAlign = TI.Align;
 
   T = T->getBaseElementTypeUnsafe();
 
   // The preferred alignment of member pointers is that of a pointer.
   if (T->isMemberPointerType())
     return getPreferredTypeAlign(getPointerDiffType().getTypePtr());
 
   if (!Target->allowsLargerPreferedTypeAlignment())
     return ABIAlign;
 
   if (const auto *RT = T->getAs<RecordType>()) {
     const RecordDecl *RD = RT->getDecl();
 
     // When used as part of a typedef, or together with a 'packed' attribute,
     // the 'aligned' attribute can be used to decrease alignment. Note that the
     // 'packed' case is already taken into consideration when computing the
     // alignment, we only need to handle the typedef case here.
     if (TI.AlignRequirement == AlignRequirementKind::RequiredByTypedef ||
         RD->isInvalidDecl())
       return ABIAlign;
 
     unsigned PreferredAlign = static_cast<unsigned>(
         toBits(getASTRecordLayout(RD).PreferredAlignment));
     assert(PreferredAlign >= ABIAlign &&
            "PreferredAlign should be at least as large as ABIAlign.");
     return PreferredAlign;
   }
 
   // Double (and, for targets supporting AIX `power` alignment, long double) and
   // long long should be naturally aligned (despite requiring less alignment) if
   // possible.
   if (const auto *CT = T->getAs<ComplexType>())
     T = CT->getElementType().getTypePtr();
   if (const auto *ET = T->getAs<EnumType>())
     T = ET->getDecl()->getIntegerType().getTypePtr();
   if (T->isSpecificBuiltinType(BuiltinType::Double) ||
       T->isSpecificBuiltinType(BuiltinType::LongLong) ||
       T->isSpecificBuiltinType(BuiltinType::ULongLong) ||
       (T->isSpecificBuiltinType(BuiltinType::LongDouble) &&
        Target->defaultsToAIXPowerAlignment()))
     // Don't increase the alignment if an alignment attribute was specified on a
     // typedef declaration.
     if (!TI.isAlignRequired())
       return std::max(ABIAlign, (unsigned)getTypeSize(T));
 
   return ABIAlign;
 }
 
 /// getTargetDefaultAlignForAttributeAligned - Return the default alignment
 /// for __attribute__((aligned)) on this target, to be used if no alignment
 /// value is specified.
 unsigned ASTContext::getTargetDefaultAlignForAttributeAligned() const {
   return getTargetInfo().getDefaultAlignForAttributeAligned();
 }
 
 /// getAlignOfGlobalVar - Return the alignment in bits that should be given
 /// to a global variable of the specified type.
 unsigned ASTContext::getAlignOfGlobalVar(QualType T) const {
   uint64_t TypeSize = getTypeSize(T.getTypePtr());
   return std::max(getPreferredTypeAlign(T),
                   getTargetInfo().getMinGlobalAlign(TypeSize));
 }
 
 /// getAlignOfGlobalVarInChars - Return the alignment in characters that
 /// should be given to a global variable of the specified type.
 CharUnits ASTContext::getAlignOfGlobalVarInChars(QualType T) const {
   return toCharUnitsFromBits(getAlignOfGlobalVar(T));
 }
 
 CharUnits ASTContext::getOffsetOfBaseWithVBPtr(const CXXRecordDecl *RD) const {
   CharUnits Offset = CharUnits::Zero();
   const ASTRecordLayout *Layout = &getASTRecordLayout(RD);
   while (const CXXRecordDecl *Base = Layout->getBaseSharingVBPtr()) {
     Offset += Layout->getBaseClassOffset(Base);
     Layout = &getASTRecordLayout(Base);
   }
   return Offset;
 }
 
 CharUnits ASTContext::getMemberPointerPathAdjustment(const APValue &MP) const {
   const ValueDecl *MPD = MP.getMemberPointerDecl();
   CharUnits ThisAdjustment = CharUnits::Zero();
   ArrayRef<const CXXRecordDecl*> Path = MP.getMemberPointerPath();
   bool DerivedMember = MP.isMemberPointerToDerivedMember();
   const CXXRecordDecl *RD = cast<CXXRecordDecl>(MPD->getDeclContext());
   for (unsigned I = 0, N = Path.size(); I != N; ++I) {
     const CXXRecordDecl *Base = RD;
     const CXXRecordDecl *Derived = Path[I];
     if (DerivedMember)
       std::swap(Base, Derived);
     ThisAdjustment += getASTRecordLayout(Derived).getBaseClassOffset(Base);
     RD = Path[I];
   }
   if (DerivedMember)
     ThisAdjustment = -ThisAdjustment;
   return ThisAdjustment;
 }
 
 /// DeepCollectObjCIvars -
 /// This routine first collects all declared, but not synthesized, ivars in
 /// super class and then collects all ivars, including those synthesized for
 /// current class. This routine is used for implementation of current class
 /// when all ivars, declared and synthesized are known.
 void ASTContext::DeepCollectObjCIvars(const ObjCInterfaceDecl *OI,
                                       bool leafClass,
                             SmallVectorImpl<const ObjCIvarDecl*> &Ivars) const {
   if (const ObjCInterfaceDecl *SuperClass = OI->getSuperClass())
     DeepCollectObjCIvars(SuperClass, false, Ivars);
   if (!leafClass) {
     for (const auto *I : OI->ivars())
       Ivars.push_back(I);
   } else {
     auto *IDecl = const_cast<ObjCInterfaceDecl *>(OI);
     for (const ObjCIvarDecl *Iv = IDecl->all_declared_ivar_begin(); Iv;
          Iv= Iv->getNextIvar())
       Ivars.push_back(Iv);
   }
 }
 
 /// CollectInheritedProtocols - Collect all protocols in current class and
 /// those inherited by it.
 void ASTContext::CollectInheritedProtocols(const Decl *CDecl,
                           llvm::SmallPtrSet<ObjCProtocolDecl*, 8> &Protocols) {
   if (const auto *OI = dyn_cast<ObjCInterfaceDecl>(CDecl)) {
     // We can use protocol_iterator here instead of
     // all_referenced_protocol_iterator since we are walking all categories.
     for (auto *Proto : OI->all_referenced_protocols()) {
       CollectInheritedProtocols(Proto, Protocols);
     }
 
     // Categories of this Interface.
     for (const auto *Cat : OI->visible_categories())
       CollectInheritedProtocols(Cat, Protocols);
 
     if (ObjCInterfaceDecl *SD = OI->getSuperClass())
       while (SD) {
         CollectInheritedProtocols(SD, Protocols);
         SD = SD->getSuperClass();
       }
   } else if (const auto *OC = dyn_cast<ObjCCategoryDecl>(CDecl)) {
     for (auto *Proto : OC->protocols()) {
       CollectInheritedProtocols(Proto, Protocols);
     }
   } else if (const auto *OP = dyn_cast<ObjCProtocolDecl>(CDecl)) {
     // Insert the protocol.
     if (!Protocols.insert(
           const_cast<ObjCProtocolDecl *>(OP->getCanonicalDecl())).second)
       return;
 
     for (auto *Proto : OP->protocols())
       CollectInheritedProtocols(Proto, Protocols);
   }
 }
 
 static bool unionHasUniqueObjectRepresentations(const ASTContext &Context,
                                                 const RecordDecl *RD) {
   assert(RD->isUnion() && "Must be union type");
   CharUnits UnionSize = Context.getTypeSizeInChars(RD->getTypeForDecl());
 
   for (const auto *Field : RD->fields()) {
     if (!Context.hasUniqueObjectRepresentations(Field->getType()))
       return false;
     CharUnits FieldSize = Context.getTypeSizeInChars(Field->getType());
     if (FieldSize != UnionSize)
       return false;
   }
   return !RD->field_empty();
 }
 
 static int64_t getSubobjectOffset(const FieldDecl *Field,
                                   const ASTContext &Context,
                                   const clang::ASTRecordLayout & /*Layout*/) {
   return Context.getFieldOffset(Field);
 }
 
 static int64_t getSubobjectOffset(const CXXRecordDecl *RD,
                                   const ASTContext &Context,
                                   const clang::ASTRecordLayout &Layout) {
   return Context.toBits(Layout.getBaseClassOffset(RD));
 }
 
 static llvm::Optional<int64_t>
 structHasUniqueObjectRepresentations(const ASTContext &Context,
                                      const RecordDecl *RD);
 
 static llvm::Optional<int64_t>
 getSubobjectSizeInBits(const FieldDecl *Field, const ASTContext &Context) {
   if (Field->getType()->isRecordType()) {
     const RecordDecl *RD = Field->getType()->getAsRecordDecl();
     if (!RD->isUnion())
       return structHasUniqueObjectRepresentations(Context, RD);
   }
   if (!Field->getType()->isReferenceType() &&
       !Context.hasUniqueObjectRepresentations(Field->getType()))
     return llvm::None;
 
   int64_t FieldSizeInBits =
       Context.toBits(Context.getTypeSizeInChars(Field->getType()));
   if (Field->isBitField()) {
     int64_t BitfieldSize = Field->getBitWidthValue(Context);
     if (BitfieldSize > FieldSizeInBits)
       return llvm::None;
     FieldSizeInBits = BitfieldSize;
   }
   return FieldSizeInBits;
 }
 
 static llvm::Optional<int64_t>
 getSubobjectSizeInBits(const CXXRecordDecl *RD, const ASTContext &Context) {
   return structHasUniqueObjectRepresentations(Context, RD);
 }
 
 template <typename RangeT>
 static llvm::Optional<int64_t> structSubobjectsHaveUniqueObjectRepresentations(
     const RangeT &Subobjects, int64_t CurOffsetInBits,
     const ASTContext &Context, const clang::ASTRecordLayout &Layout) {
   for (const auto *Subobject : Subobjects) {
     llvm::Optional<int64_t> SizeInBits =
         getSubobjectSizeInBits(Subobject, Context);
     if (!SizeInBits)
       return llvm::None;
     if (*SizeInBits != 0) {
       int64_t Offset = getSubobjectOffset(Subobject, Context, Layout);
       if (Offset != CurOffsetInBits)
         return llvm::None;
       CurOffsetInBits += *SizeInBits;
     }
   }
   return CurOffsetInBits;
 }
 
 static llvm::Optional<int64_t>
 structHasUniqueObjectRepresentations(const ASTContext &Context,
                                      const RecordDecl *RD) {
   assert(!RD->isUnion() && "Must be struct/class type");
   const auto &Layout = Context.getASTRecordLayout(RD);
 
   int64_t CurOffsetInBits = 0;
   if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RD)) {
     if (ClassDecl->isDynamicClass())
       return llvm::None;
 
     SmallVector<CXXRecordDecl *, 4> Bases;
     for (const auto &Base : ClassDecl->bases()) {
       // Empty types can be inherited from, and non-empty types can potentially
       // have tail padding, so just make sure there isn't an error.
       Bases.emplace_back(Base.getType()->getAsCXXRecordDecl());
     }
 
     llvm::sort(Bases, [&](const CXXRecordDecl *L, const CXXRecordDecl *R) {
       return Layout.getBaseClassOffset(L) < Layout.getBaseClassOffset(R);
     });
 
     llvm::Optional<int64_t> OffsetAfterBases =
         structSubobjectsHaveUniqueObjectRepresentations(Bases, CurOffsetInBits,
                                                         Context, Layout);
     if (!OffsetAfterBases)
       return llvm::None;
     CurOffsetInBits = *OffsetAfterBases;
   }
 
   llvm::Optional<int64_t> OffsetAfterFields =
       structSubobjectsHaveUniqueObjectRepresentations(
           RD->fields(), CurOffsetInBits, Context, Layout);
   if (!OffsetAfterFields)
     return llvm::None;
   CurOffsetInBits = *OffsetAfterFields;
 
   return CurOffsetInBits;
 }
 
 bool ASTContext::hasUniqueObjectRepresentations(QualType Ty) const {
   // C++17 [meta.unary.prop]:
   //   The predicate condition for a template specialization
   //   has_unique_object_representations<T> shall be
   //   satisfied if and only if:
   //     (9.1) - T is trivially copyable, and
   //     (9.2) - any two objects of type T with the same value have the same
   //     object representation, where two objects
   //   of array or non-union class type are considered to have the same value
   //   if their respective sequences of
   //   direct subobjects have the same values, and two objects of union type
   //   are considered to have the same
   //   value if they have the same active member and the corresponding members
   //   have the same value.
   //   The set of scalar types for which this condition holds is
   //   implementation-defined. [ Note: If a type has padding
   //   bits, the condition does not hold; otherwise, the condition holds true
   //   for unsigned integral types. -- end note ]
   assert(!Ty.isNull() && "Null QualType sent to unique object rep check");
 
   // Arrays are unique only if their element type is unique.
   if (Ty->isArrayType())
     return hasUniqueObjectRepresentations(getBaseElementType(Ty));
 
   // (9.1) - T is trivially copyable...
   if (!Ty.isTriviallyCopyableType(*this))
     return false;
 
   // All integrals and enums are unique.
   if (Ty->isIntegralOrEnumerationType())
     return true;
 
   // All other pointers are unique.
   if (Ty->isPointerType())
     return true;
 
   if (Ty->isMemberPointerType()) {
     const auto *MPT = Ty->getAs<MemberPointerType>();
     return !ABI->getMemberPointerInfo(MPT).HasPadding;
   }
 
   if (Ty->isRecordType()) {
     const RecordDecl *Record = Ty->castAs<RecordType>()->getDecl();
 
     if (Record->isInvalidDecl())
       return false;
 
     if (Record->isUnion())
       return unionHasUniqueObjectRepresentations(*this, Record);
 
     Optional<int64_t> StructSize =
         structHasUniqueObjectRepresentations(*this, Record);
 
     return StructSize &&
            StructSize.getValue() == static_cast<int64_t>(getTypeSize(Ty));
   }
 
   // FIXME: More cases to handle here (list by rsmith):
   // vectors (careful about, eg, vector of 3 foo)
   // _Complex int and friends
   // _Atomic T
   // Obj-C block pointers
   // Obj-C object pointers
   // and perhaps OpenCL's various builtin types (pipe, sampler_t, event_t,
   // clk_event_t, queue_t, reserve_id_t)
   // There're also Obj-C class types and the Obj-C selector type, but I think it
   // makes sense for those to return false here.
 
   return false;
 }
 
 unsigned ASTContext::CountNonClassIvars(const ObjCInterfaceDecl *OI) const {
   unsigned count = 0;
   // Count ivars declared in class extension.
   for (const auto *Ext : OI->known_extensions())
     count += Ext->ivar_size();
 
   // Count ivar defined in this class's implementation.  This
   // includes synthesized ivars.
   if (ObjCImplementationDecl *ImplDecl = OI->getImplementation())
     count += ImplDecl->ivar_size();
 
   return count;
 }
 
 bool ASTContext::isSentinelNullExpr(const Expr *E) {
   if (!E)
     return false;
 
   // nullptr_t is always treated as null.
   if (E->getType()->isNullPtrType()) return true;
 
   if (E->getType()->isAnyPointerType() &&
       E->IgnoreParenCasts()->isNullPointerConstant(*this,
                                                 Expr::NPC_ValueDependentIsNull))
     return true;
 
   // Unfortunately, __null has type 'int'.
   if (isa<GNUNullExpr>(E)) return true;
 
   return false;
 }
 
 /// Get the implementation of ObjCInterfaceDecl, or nullptr if none
 /// exists.
 ObjCImplementationDecl *ASTContext::getObjCImplementation(ObjCInterfaceDecl *D) {
   llvm::DenseMap<ObjCContainerDecl*, ObjCImplDecl*>::iterator
     I = ObjCImpls.find(D);
   if (I != ObjCImpls.end())
     return cast<ObjCImplementationDecl>(I->second);
   return nullptr;
 }
 
 /// Get the implementation of ObjCCategoryDecl, or nullptr if none
 /// exists.
 ObjCCategoryImplDecl *ASTContext::getObjCImplementation(ObjCCategoryDecl *D) {
   llvm::DenseMap<ObjCContainerDecl*, ObjCImplDecl*>::iterator
     I = ObjCImpls.find(D);
   if (I != ObjCImpls.end())
     return cast<ObjCCategoryImplDecl>(I->second);
   return nullptr;
 }
 
 /// Set the implementation of ObjCInterfaceDecl.
 void ASTContext::setObjCImplementation(ObjCInterfaceDecl *IFaceD,
                            ObjCImplementationDecl *ImplD) {
   assert(IFaceD && ImplD && "Passed null params");
   ObjCImpls[IFaceD] = ImplD;
 }
 
 /// Set the implementation of ObjCCategoryDecl.
 void ASTContext::setObjCImplementation(ObjCCategoryDecl *CatD,
                            ObjCCategoryImplDecl *ImplD) {
   assert(CatD && ImplD && "Passed null params");
   ObjCImpls[CatD] = ImplD;
 }
 
 const ObjCMethodDecl *
 ASTContext::getObjCMethodRedeclaration(const ObjCMethodDecl *MD) const {
   return ObjCMethodRedecls.lookup(MD);
 }
 
 void ASTContext::setObjCMethodRedeclaration(const ObjCMethodDecl *MD,
                                             const ObjCMethodDecl *Redecl) {
   assert(!getObjCMethodRedeclaration(MD) && "MD already has a redeclaration");
   ObjCMethodRedecls[MD] = Redecl;
 }
 
 const ObjCInterfaceDecl *ASTContext::getObjContainingInterface(
                                               const NamedDecl *ND) const {
   if (const auto *ID = dyn_cast<ObjCInterfaceDecl>(ND->getDeclContext()))
     return ID;
   if (const auto *CD = dyn_cast<ObjCCategoryDecl>(ND->getDeclContext()))
     return CD->getClassInterface();
   if (const auto *IMD = dyn_cast<ObjCImplDecl>(ND->getDeclContext()))
     return IMD->getClassInterface();
 
   return nullptr;
 }
 
 /// Get the copy initialization expression of VarDecl, or nullptr if
 /// none exists.
 BlockVarCopyInit ASTContext::getBlockVarCopyInit(const VarDecl *VD) const {
   assert(VD && "Passed null params");
   assert(VD->hasAttr<BlocksAttr>() &&
          "getBlockVarCopyInits - not __block var");
   auto I = BlockVarCopyInits.find(VD);
   if (I != BlockVarCopyInits.end())
     return I->second;
   return {nullptr, false};
 }
 
 /// Set the copy initialization expression of a block var decl.
 void ASTContext::setBlockVarCopyInit(const VarDecl*VD, Expr *CopyExpr,
                                      bool CanThrow) {
   assert(VD && CopyExpr && "Passed null params");
   assert(VD->hasAttr<BlocksAttr>() &&
          "setBlockVarCopyInits - not __block var");
   BlockVarCopyInits[VD].setExprAndFlag(CopyExpr, CanThrow);
 }
 
 TypeSourceInfo *ASTContext::CreateTypeSourceInfo(QualType T,
                                                  unsigned DataSize) const {
   if (!DataSize)
     DataSize = TypeLoc::getFullDataSizeForType(T);
   else
     assert(DataSize == TypeLoc::getFullDataSizeForType(T) &&
            "incorrect data size provided to CreateTypeSourceInfo!");
 
   auto *TInfo =
     (TypeSourceInfo*)BumpAlloc.Allocate(sizeof(TypeSourceInfo) + DataSize, 8);
   new (TInfo) TypeSourceInfo(T);
   return TInfo;
 }
 
 TypeSourceInfo *ASTContext::getTrivialTypeSourceInfo(QualType T,
                                                      SourceLocation L) const {
   TypeSourceInfo *DI = CreateTypeSourceInfo(T);
   DI->getTypeLoc().initialize(const_cast<ASTContext &>(*this), L);
   return DI;
 }
 
 const ASTRecordLayout &
 ASTContext::getASTObjCInterfaceLayout(const ObjCInterfaceDecl *D) const {
   return getObjCLayout(D, nullptr);
 }
 
 const ASTRecordLayout &
 ASTContext::getASTObjCImplementationLayout(
                                         const ObjCImplementationDecl *D) const {
   return getObjCLayout(D->getClassInterface(), D);
 }
 
 //===----------------------------------------------------------------------===//
 //                   Type creation/memoization methods
 //===----------------------------------------------------------------------===//
 
 QualType
 ASTContext::getExtQualType(const Type *baseType, Qualifiers quals) const {
   unsigned fastQuals = quals.getFastQualifiers();
   quals.removeFastQualifiers();
 
   // Check if we've already instantiated this type.
   llvm::FoldingSetNodeID ID;
   ExtQuals::Profile(ID, baseType, quals);
   void *insertPos = nullptr;
   if (ExtQuals *eq = ExtQualNodes.FindNodeOrInsertPos(ID, insertPos)) {
     assert(eq->getQualifiers() == quals);
     return QualType(eq, fastQuals);
   }
 
   // If the base type is not canonical, make the appropriate canonical type.
   QualType canon;
   if (!baseType->isCanonicalUnqualified()) {
     SplitQualType canonSplit = baseType->getCanonicalTypeInternal().split();
     canonSplit.Quals.addConsistentQualifiers(quals);
     canon = getExtQualType(canonSplit.Ty, canonSplit.Quals);
 
     // Re-find the insert position.
     (void) ExtQualNodes.FindNodeOrInsertPos(ID, insertPos);
   }
 
   auto *eq = new (*this, TypeAlignment) ExtQuals(baseType, canon, quals);
   ExtQualNodes.InsertNode(eq, insertPos);
   return QualType(eq, fastQuals);
 }
 
 QualType ASTContext::getAddrSpaceQualType(QualType T,
                                           LangAS AddressSpace) const {
   QualType CanT = getCanonicalType(T);
   if (CanT.getAddressSpace() == AddressSpace)
     return T;
 
   // If we are composing extended qualifiers together, merge together
   // into one ExtQuals node.
   QualifierCollector Quals;
   const Type *TypeNode = Quals.strip(T);
 
   // If this type already has an address space specified, it cannot get
   // another one.
   assert(!Quals.hasAddressSpace() &&
          "Type cannot be in multiple addr spaces!");
   Quals.addAddressSpace(AddressSpace);
 
   return getExtQualType(TypeNode, Quals);
 }
 
 QualType ASTContext::removeAddrSpaceQualType(QualType T) const {
   // If the type is not qualified with an address space, just return it
   // immediately.
   if (!T.hasAddressSpace())
     return T;
 
   // If we are composing extended qualifiers together, merge together
   // into one ExtQuals node.
   QualifierCollector Quals;
   const Type *TypeNode;
 
   while (T.hasAddressSpace()) {
     TypeNode = Quals.strip(T);
 
     // If the type no longer has an address space after stripping qualifiers,
     // jump out.
     if (!QualType(TypeNode, 0).hasAddressSpace())
       break;
 
     // There might be sugar in the way. Strip it and try again.
     T = T.getSingleStepDesugaredType(*this);
   }
 
   Quals.removeAddressSpace();
 
   // Removal of the address space can mean there are no longer any
   // non-fast qualifiers, so creating an ExtQualType isn't possible (asserts)
   // or required.
   if (Quals.hasNonFastQualifiers())
     return getExtQualType(TypeNode, Quals);
   else
     return QualType(TypeNode, Quals.getFastQualifiers());
 }
 
 QualType ASTContext::getObjCGCQualType(QualType T,
                                        Qualifiers::GC GCAttr) const {
   QualType CanT = getCanonicalType(T);
   if (CanT.getObjCGCAttr() == GCAttr)
     return T;
 
   if (const auto *ptr = T->getAs<PointerType>()) {
     QualType Pointee = ptr->getPointeeType();
     if (Pointee->isAnyPointerType()) {
       QualType ResultType = getObjCGCQualType(Pointee, GCAttr);
       return getPointerType(ResultType);
     }
   }
 
   // If we are composing extended qualifiers together, merge together
   // into one ExtQuals node.
   QualifierCollector Quals;
   const Type *TypeNode = Quals.strip(T);
 
   // If this type already has an ObjCGC specified, it cannot get
   // another one.
   assert(!Quals.hasObjCGCAttr() &&
          "Type cannot have multiple ObjCGCs!");
   Quals.addObjCGCAttr(GCAttr);
 
   return getExtQualType(TypeNode, Quals);
 }
 
 QualType ASTContext::removePtrSizeAddrSpace(QualType T) const {
   if (const PointerType *Ptr = T->getAs<PointerType>()) {
     QualType Pointee = Ptr->getPointeeType();
     if (isPtrSizeAddressSpace(Pointee.getAddressSpace())) {
       return getPointerType(removeAddrSpaceQualType(Pointee));
     }
   }
   return T;
 }
 
 const FunctionType *ASTContext::adjustFunctionType(const FunctionType *T,
                                                    FunctionType::ExtInfo Info) {
   if (T->getExtInfo() == Info)
     return T;
 
   QualType Result;
   if (const auto *FNPT = dyn_cast<FunctionNoProtoType>(T)) {
     Result = getFunctionNoProtoType(FNPT->getReturnType(), Info);
   } else {
     const auto *FPT = cast<FunctionProtoType>(T);
     FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo();
     EPI.ExtInfo = Info;
     Result = getFunctionType(FPT->getReturnType(), FPT->getParamTypes(), EPI);
   }
 
   return cast<FunctionType>(Result.getTypePtr());
 }
 
 void ASTContext::adjustDeducedFunctionResultType(FunctionDecl *FD,
                                                  QualType ResultType) {
   FD = FD->getMostRecentDecl();
   while (true) {
     const auto *FPT = FD->getType()->castAs<FunctionProtoType>();
     FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo();
     FD->setType(getFunctionType(ResultType, FPT->getParamTypes(), EPI));
     if (FunctionDecl *Next = FD->getPreviousDecl())
       FD = Next;
     else
       break;
   }
   if (ASTMutationListener *L = getASTMutationListener())
     L->DeducedReturnType(FD, ResultType);
 }
 
 /// Get a function type and produce the equivalent function type with the
 /// specified exception specification. Type sugar that can be present on a
 /// declaration of a function with an exception specification is permitted
 /// and preserved. Other type sugar (for instance, typedefs) is not.
 QualType ASTContext::getFunctionTypeWithExceptionSpec(
     QualType Orig, const FunctionProtoType::ExceptionSpecInfo &ESI) {
   // Might have some parens.
   if (const auto *PT = dyn_cast<ParenType>(Orig))
     return getParenType(
         getFunctionTypeWithExceptionSpec(PT->getInnerType(), ESI));
 
   // Might be wrapped in a macro qualified type.
   if (const auto *MQT = dyn_cast<MacroQualifiedType>(Orig))
     return getMacroQualifiedType(
         getFunctionTypeWithExceptionSpec(MQT->getUnderlyingType(), ESI),
         MQT->getMacroIdentifier());
 
   // Might have a calling-convention attribute.
   if (const auto *AT = dyn_cast<AttributedType>(Orig))
     return getAttributedType(
         AT->getAttrKind(),
         getFunctionTypeWithExceptionSpec(AT->getModifiedType(), ESI),
         getFunctionTypeWithExceptionSpec(AT->getEquivalentType(), ESI));
 
   // Anything else must be a function type. Rebuild it with the new exception
   // specification.
   const auto *Proto = Orig->castAs<FunctionProtoType>();
   return getFunctionType(
       Proto->getReturnType(), Proto->getParamTypes(),
       Proto->getExtProtoInfo().withExceptionSpec(ESI));
 }
 
 bool ASTContext::hasSameFunctionTypeIgnoringExceptionSpec(QualType T,
                                                           QualType U) {
   return hasSameType(T, U) ||
          (getLangOpts().CPlusPlus17 &&
           hasSameType(getFunctionTypeWithExceptionSpec(T, EST_None),
                       getFunctionTypeWithExceptionSpec(U, EST_None)));
 }
 
 QualType ASTContext::getFunctionTypeWithoutPtrSizes(QualType T) {
   if (const auto *Proto = T->getAs<FunctionProtoType>()) {
     QualType RetTy = removePtrSizeAddrSpace(Proto->getReturnType());
     SmallVector<QualType, 16> Args(Proto->param_types());
     for (unsigned i = 0, n = Args.size(); i != n; ++i)
       Args[i] = removePtrSizeAddrSpace(Args[i]);
     return getFunctionType(RetTy, Args, Proto->getExtProtoInfo());
   }
 
   if (const FunctionNoProtoType *Proto = T->getAs<FunctionNoProtoType>()) {
     QualType RetTy = removePtrSizeAddrSpace(Proto->getReturnType());
     return getFunctionNoProtoType(RetTy, Proto->getExtInfo());
   }
 
   return T;
 }
 
 bool ASTContext::hasSameFunctionTypeIgnoringPtrSizes(QualType T, QualType U) {
   return hasSameType(T, U) ||
          hasSameType(getFunctionTypeWithoutPtrSizes(T),
                      getFunctionTypeWithoutPtrSizes(U));
 }
 
 void ASTContext::adjustExceptionSpec(
     FunctionDecl *FD, const FunctionProtoType::ExceptionSpecInfo &ESI,
     bool AsWritten) {
   // Update the type.
   QualType Updated =
       getFunctionTypeWithExceptionSpec(FD->getType(), ESI);
   FD->setType(Updated);
 
   if (!AsWritten)
     return;
 
   // Update the type in the type source information too.
   if (TypeSourceInfo *TSInfo = FD->getTypeSourceInfo()) {
     // If the type and the type-as-written differ, we may need to update
     // the type-as-written too.
     if (TSInfo->getType() != FD->getType())
       Updated = getFunctionTypeWithExceptionSpec(TSInfo->getType(), ESI);
 
     // FIXME: When we get proper type location information for exceptions,
     // we'll also have to rebuild the TypeSourceInfo. For now, we just patch
     // up the TypeSourceInfo;
     assert(TypeLoc::getFullDataSizeForType(Updated) ==
                TypeLoc::getFullDataSizeForType(TSInfo->getType()) &&
            "TypeLoc size mismatch from updating exception specification");
     TSInfo->overrideType(Updated);
   }
 }
 
 /// getComplexType - Return the uniqued reference to the type for a complex
 /// number with the specified element type.
 QualType ASTContext::getComplexType(QualType T) const {
   // Unique pointers, to guarantee there is only one pointer of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
   ComplexType::Profile(ID, T);
 
   void *InsertPos = nullptr;
   if (ComplexType *CT = ComplexTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(CT, 0);
 
   // If the pointee type isn't canonical, this won't be a canonical type either,
   // so fill in the canonical type field.
   QualType Canonical;
   if (!T.isCanonical()) {
     Canonical = getComplexType(getCanonicalType(T));
 
     // Get the new insert position for the node we care about.
     ComplexType *NewIP = ComplexTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
   auto *New = new (*this, TypeAlignment) ComplexType(T, Canonical);
   Types.push_back(New);
   ComplexTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
 }
 
 /// getPointerType - Return the uniqued reference to the type for a pointer to
 /// the specified type.
 QualType ASTContext::getPointerType(QualType T) const {
   // Unique pointers, to guarantee there is only one pointer of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
   PointerType::Profile(ID, T);
 
   void *InsertPos = nullptr;
   if (PointerType *PT = PointerTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(PT, 0);
 
   // If the pointee type isn't canonical, this won't be a canonical type either,
   // so fill in the canonical type field.
   QualType Canonical;
   if (!T.isCanonical()) {
     Canonical = getPointerType(getCanonicalType(T));
 
     // Get the new insert position for the node we care about.
     PointerType *NewIP = PointerTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
   auto *New = new (*this, TypeAlignment) PointerType(T, Canonical);
   Types.push_back(New);
   PointerTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
 }
 
 QualType ASTContext::getAdjustedType(QualType Orig, QualType New) const {
   llvm::FoldingSetNodeID ID;
   AdjustedType::Profile(ID, Orig, New);
   void *InsertPos = nullptr;
   AdjustedType *AT = AdjustedTypes.FindNodeOrInsertPos(ID, InsertPos);
   if (AT)
     return QualType(AT, 0);
 
   QualType Canonical = getCanonicalType(New);
 
   // Get the new insert position for the node we care about.
   AT = AdjustedTypes.FindNodeOrInsertPos(ID, InsertPos);
   assert(!AT && "Shouldn't be in the map!");
 
   AT = new (*this, TypeAlignment)
       AdjustedType(Type::Adjusted, Orig, New, Canonical);
   Types.push_back(AT);
   AdjustedTypes.InsertNode(AT, InsertPos);
   return QualType(AT, 0);
 }
 
 QualType ASTContext::getDecayedType(QualType T) const {
   assert((T->isArrayType() || T->isFunctionType()) && "T does not decay");
 
   QualType Decayed;
 
   // C99 6.7.5.3p7:
   //   A declaration of a parameter as "array of type" shall be
   //   adjusted to "qualified pointer to type", where the type
   //   qualifiers (if any) are those specified within the [ and ] of
   //   the array type derivation.
   if (T->isArrayType())
     Decayed = getArrayDecayedType(T);
 
   // C99 6.7.5.3p8:
   //   A declaration of a parameter as "function returning type"
   //   shall be adjusted to "pointer to function returning type", as
   //   in 6.3.2.1.
   if (T->isFunctionType())
     Decayed = getPointerType(T);
 
   llvm::FoldingSetNodeID ID;
   AdjustedType::Profile(ID, T, Decayed);
   void *InsertPos = nullptr;
   AdjustedType *AT = AdjustedTypes.FindNodeOrInsertPos(ID, InsertPos);
   if (AT)
     return QualType(AT, 0);
 
   QualType Canonical = getCanonicalType(Decayed);
 
   // Get the new insert position for the node we care about.
   AT = AdjustedTypes.FindNodeOrInsertPos(ID, InsertPos);
   assert(!AT && "Shouldn't be in the map!");
 
   AT = new (*this, TypeAlignment) DecayedType(T, Decayed, Canonical);
   Types.push_back(AT);
   AdjustedTypes.InsertNode(AT, InsertPos);
   return QualType(AT, 0);
 }
 
 /// getBlockPointerType - Return the uniqued reference to the type for
 /// a pointer to the specified block.
 QualType ASTContext::getBlockPointerType(QualType T) const {
   assert(T->isFunctionType() && "block of function types only");
   // Unique pointers, to guarantee there is only one block of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
   BlockPointerType::Profile(ID, T);
 
   void *InsertPos = nullptr;
   if (BlockPointerType *PT =
         BlockPointerTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(PT, 0);
 
   // If the block pointee type isn't canonical, this won't be a canonical
   // type either so fill in the canonical type field.
   QualType Canonical;
   if (!T.isCanonical()) {
     Canonical = getBlockPointerType(getCanonicalType(T));
 
     // Get the new insert position for the node we care about.
     BlockPointerType *NewIP =
       BlockPointerTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
   auto *New = new (*this, TypeAlignment) BlockPointerType(T, Canonical);
   Types.push_back(New);
   BlockPointerTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
 }
 
 /// getLValueReferenceType - Return the uniqued reference to the type for an
 /// lvalue reference to the specified type.
 QualType
 ASTContext::getLValueReferenceType(QualType T, bool SpelledAsLValue) const {
   assert((!T->isPlaceholderType() ||
           T->isSpecificPlaceholderType(BuiltinType::UnknownAny)) &&
          "Unresolved placeholder type");
 
   // Unique pointers, to guarantee there is only one pointer of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
   ReferenceType::Profile(ID, T, SpelledAsLValue);
 
   void *InsertPos = nullptr;
   if (LValueReferenceType *RT =
         LValueReferenceTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(RT, 0);
 
   const auto *InnerRef = T->getAs<ReferenceType>();
 
   // If the referencee type isn't canonical, this won't be a canonical type
   // either, so fill in the canonical type field.
   QualType Canonical;
   if (!SpelledAsLValue || InnerRef || !T.isCanonical()) {
     QualType PointeeType = (InnerRef ? InnerRef->getPointeeType() : T);
     Canonical = getLValueReferenceType(getCanonicalType(PointeeType));
 
     // Get the new insert position for the node we care about.
     LValueReferenceType *NewIP =
       LValueReferenceTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
 
   auto *New = new (*this, TypeAlignment) LValueReferenceType(T, Canonical,
                                                              SpelledAsLValue);
   Types.push_back(New);
   LValueReferenceTypes.InsertNode(New, InsertPos);
 
   return QualType(New, 0);
 }
 
 /// getRValueReferenceType - Return the uniqued reference to the type for an
 /// rvalue reference to the specified type.
 QualType ASTContext::getRValueReferenceType(QualType T) const {
   assert((!T->isPlaceholderType() ||
           T->isSpecificPlaceholderType(BuiltinType::UnknownAny)) &&
          "Unresolved placeholder type");
 
   // Unique pointers, to guarantee there is only one pointer of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
   ReferenceType::Profile(ID, T, false);
 
   void *InsertPos = nullptr;
   if (RValueReferenceType *RT =
         RValueReferenceTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(RT, 0);
 
   const auto *InnerRef = T->getAs<ReferenceType>();
 
   // If the referencee type isn't canonical, this won't be a canonical type
   // either, so fill in the canonical type field.
   QualType Canonical;
   if (InnerRef || !T.isCanonical()) {
     QualType PointeeType = (InnerRef ? InnerRef->getPointeeType() : T);
     Canonical = getRValueReferenceType(getCanonicalType(PointeeType));
 
     // Get the new insert position for the node we care about.
     RValueReferenceType *NewIP =
       RValueReferenceTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
 
   auto *New = new (*this, TypeAlignment) RValueReferenceType(T, Canonical);
   Types.push_back(New);
   RValueReferenceTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
 }
 
 /// getMemberPointerType - Return the uniqued reference to the type for a
 /// member pointer to the specified type, in the specified class.
 QualType ASTContext::getMemberPointerType(QualType T, const Type *Cls) const {
   // Unique pointers, to guarantee there is only one pointer of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
   MemberPointerType::Profile(ID, T, Cls);
 
   void *InsertPos = nullptr;
   if (MemberPointerType *PT =
       MemberPointerTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(PT, 0);
 
   // If the pointee or class type isn't canonical, this won't be a canonical
   // type either, so fill in the canonical type field.
   QualType Canonical;
   if (!T.isCanonical() || !Cls->isCanonicalUnqualified()) {
     Canonical = getMemberPointerType(getCanonicalType(T),getCanonicalType(Cls));
 
     // Get the new insert position for the node we care about.
     MemberPointerType *NewIP =
       MemberPointerTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
   auto *New = new (*this, TypeAlignment) MemberPointerType(T, Cls, Canonical);
   Types.push_back(New);
   MemberPointerTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
 }
 
 /// getConstantArrayType - Return the unique reference to the type for an
 /// array of the specified element type.
 QualType ASTContext::getConstantArrayType(QualType EltTy,
                                           const llvm::APInt &ArySizeIn,
                                           const Expr *SizeExpr,
                                           ArrayType::ArraySizeModifier ASM,
                                           unsigned IndexTypeQuals) const {
   assert((EltTy->isDependentType() ||
           EltTy->isIncompleteType() || EltTy->isConstantSizeType()) &&
          "Constant array of VLAs is illegal!");
 
   // We only need the size as part of the type if it's instantiation-dependent.
   if (SizeExpr && !SizeExpr->isInstantiationDependent())
     SizeExpr = nullptr;
 
   // Convert the array size into a canonical width matching the pointer size for
   // the target.
   llvm::APInt ArySize(ArySizeIn);
   ArySize = ArySize.zextOrTrunc(Target->getMaxPointerWidth());
 
   llvm::FoldingSetNodeID ID;
   ConstantArrayType::Profile(ID, *this, EltTy, ArySize, SizeExpr, ASM,
                              IndexTypeQuals);
 
   void *InsertPos = nullptr;
   if (ConstantArrayType *ATP =
       ConstantArrayTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(ATP, 0);
 
   // If the element type isn't canonical or has qualifiers, or the array bound
   // is instantiation-dependent, this won't be a canonical type either, so fill
   // in the canonical type field.
   QualType Canon;
   if (!EltTy.isCanonical() || EltTy.hasLocalQualifiers() || SizeExpr) {
     SplitQualType canonSplit = getCanonicalType(EltTy).split();
     Canon = getConstantArrayType(QualType(canonSplit.Ty, 0), ArySize, nullptr,
                                  ASM, IndexTypeQuals);
     Canon = getQualifiedType(Canon, canonSplit.Quals);
 
     // Get the new insert position for the node we care about.
     ConstantArrayType *NewIP =
       ConstantArrayTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
 
   void *Mem = Allocate(
       ConstantArrayType::totalSizeToAlloc<const Expr *>(SizeExpr ? 1 : 0),
       TypeAlignment);
   auto *New = new (Mem)
     ConstantArrayType(EltTy, Canon, ArySize, SizeExpr, ASM, IndexTypeQuals);
   ConstantArrayTypes.InsertNode(New, InsertPos);
   Types.push_back(New);
   return QualType(New, 0);
 }
 
 /// getVariableArrayDecayedType - Turns the given type, which may be
 /// variably-modified, into the corresponding type with all the known
 /// sizes replaced with [*].
 QualType ASTContext::getVariableArrayDecayedType(QualType type) const {
   // Vastly most common case.
   if (!type->isVariablyModifiedType()) return type;
 
   QualType result;
 
   SplitQualType split = type.getSplitDesugaredType();
   const Type *ty = split.Ty;
   switch (ty->getTypeClass()) {
 #define TYPE(Class, Base)
 #define ABSTRACT_TYPE(Class, Base)
 #define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
 #include "clang/AST/TypeNodes.inc"
     llvm_unreachable("didn't desugar past all non-canonical types?");
 
   // These types should never be variably-modified.
   case Type::Builtin:
   case Type::Complex:
   case Type::Vector:
   case Type::DependentVector:
   case Type::ExtVector:
   case Type::DependentSizedExtVector:
   case Type::ConstantMatrix:
   case Type::DependentSizedMatrix:
   case Type::DependentAddressSpace:
   case Type::ObjCObject:
   case Type::ObjCInterface:
   case Type::ObjCObjectPointer:
   case Type::Record:
   case Type::Enum:
   case Type::UnresolvedUsing:
   case Type::TypeOfExpr:
   case Type::TypeOf:
   case Type::Decltype:
   case Type::UnaryTransform:
   case Type::DependentName:
   case Type::InjectedClassName:
   case Type::TemplateSpecialization:
   case Type::DependentTemplateSpecialization:
   case Type::TemplateTypeParm:
   case Type::SubstTemplateTypeParmPack:
   case Type::Auto:
   case Type::DeducedTemplateSpecialization:
   case Type::PackExpansion:
   case Type::BitInt:
   case Type::DependentBitInt:
     llvm_unreachable("type should never be variably-modified");
 
   // These types can be variably-modified but should never need to
   // further decay.
   case Type::FunctionNoProto:
   case Type::FunctionProto:
   case Type::BlockPointer:
   case Type::MemberPointer:
   case Type::Pipe:
     return type;
 
   // These types can be variably-modified.  All these modifications
   // preserve structure except as noted by comments.
   // TODO: if we ever care about optimizing VLAs, there are no-op
   // optimizations available here.
   case Type::Pointer:
     result = getPointerType(getVariableArrayDecayedType(
                               cast<PointerType>(ty)->getPointeeType()));
     break;
 
   case Type::LValueReference: {
     const auto *lv = cast<LValueReferenceType>(ty);
     result = getLValueReferenceType(
                  getVariableArrayDecayedType(lv->getPointeeType()),
                                     lv->isSpelledAsLValue());
     break;
   }
 
   case Type::RValueReference: {
     const auto *lv = cast<RValueReferenceType>(ty);
     result = getRValueReferenceType(
                  getVariableArrayDecayedType(lv->getPointeeType()));
     break;
   }
 
   case Type::Atomic: {
     const auto *at = cast<AtomicType>(ty);
     result = getAtomicType(getVariableArrayDecayedType(at->getValueType()));
     break;
   }
 
   case Type::ConstantArray: {
     const auto *cat = cast<ConstantArrayType>(ty);
     result = getConstantArrayType(
                  getVariableArrayDecayedType(cat->getElementType()),
                                   cat->getSize(),
                                   cat->getSizeExpr(),
                                   cat->getSizeModifier(),
                                   cat->getIndexTypeCVRQualifiers());
     break;
   }
 
   case Type::DependentSizedArray: {
     const auto *dat = cast<DependentSizedArrayType>(ty);
     result = getDependentSizedArrayType(
                  getVariableArrayDecayedType(dat->getElementType()),
                                         dat->getSizeExpr(),
                                         dat->getSizeModifier(),
                                         dat->getIndexTypeCVRQualifiers(),
                                         dat->getBracketsRange());
     break;
   }
 
   // Turn incomplete types into [*] types.
   case Type::IncompleteArray: {
     const auto *iat = cast<IncompleteArrayType>(ty);
     result = getVariableArrayType(
                  getVariableArrayDecayedType(iat->getElementType()),
                                   /*size*/ nullptr,
                                   ArrayType::Normal,
                                   iat->getIndexTypeCVRQualifiers(),
                                   SourceRange());
     break;
   }
 
   // Turn VLA types into [*] types.
   case Type::VariableArray: {
     const auto *vat = cast<VariableArrayType>(ty);
     result = getVariableArrayType(
                  getVariableArrayDecayedType(vat->getElementType()),
                                   /*size*/ nullptr,
                                   ArrayType::Star,
                                   vat->getIndexTypeCVRQualifiers(),
                                   vat->getBracketsRange());
     break;
   }
   }
 
   // Apply the top-level qualifiers from the original.
   return getQualifiedType(result, split.Quals);
 }
 
 /// getVariableArrayType - Returns a non-unique reference to the type for a
 /// variable array of the specified element type.
 QualType ASTContext::getVariableArrayType(QualType EltTy,
                                           Expr *NumElts,
                                           ArrayType::ArraySizeModifier ASM,
                                           unsigned IndexTypeQuals,
                                           SourceRange Brackets) const {
   // Since we don't unique expressions, it isn't possible to unique VLA's
   // that have an expression provided for their size.
   QualType Canon;
 
   // Be sure to pull qualifiers off the element type.
   if (!EltTy.isCanonical() || EltTy.hasLocalQualifiers()) {
     SplitQualType canonSplit = getCanonicalType(EltTy).split();
     Canon = getVariableArrayType(QualType(canonSplit.Ty, 0), NumElts, ASM,
                                  IndexTypeQuals, Brackets);
     Canon = getQualifiedType(Canon, canonSplit.Quals);
   }
 
   auto *New = new (*this, TypeAlignment)
     VariableArrayType(EltTy, Canon, NumElts, ASM, IndexTypeQuals, Brackets);
 
   VariableArrayTypes.push_back(New);
   Types.push_back(New);
   return QualType(New, 0);
 }
 
 /// getDependentSizedArrayType - Returns a non-unique reference to
 /// the type for a dependently-sized array of the specified element
 /// type.
 QualType ASTContext::getDependentSizedArrayType(QualType elementType,
                                                 Expr *numElements,
                                                 ArrayType::ArraySizeModifier ASM,
                                                 unsigned elementTypeQuals,
                                                 SourceRange brackets) const {
   assert((!numElements || numElements->isTypeDependent() ||
           numElements->isValueDependent()) &&
          "Size must be type- or value-dependent!");
 
   // Dependently-sized array types that do not have a specified number
   // of elements will have their sizes deduced from a dependent
   // initializer.  We do no canonicalization here at all, which is okay
   // because they can't be used in most locations.
   if (!numElements) {
     auto *newType
       = new (*this, TypeAlignment)
           DependentSizedArrayType(*this, elementType, QualType(),
                                   numElements, ASM, elementTypeQuals,
                                   brackets);
     Types.push_back(newType);
     return QualType(newType, 0);
   }
 
   // Otherwise, we actually build a new type every time, but we
   // also build a canonical type.
 
   SplitQualType canonElementType = getCanonicalType(elementType).split();
 
   void *insertPos = nullptr;
   llvm::FoldingSetNodeID ID;
   DependentSizedArrayType::Profile(ID, *this,
                                    QualType(canonElementType.Ty, 0),
                                    ASM, elementTypeQuals, numElements);
 
   // Look for an existing type with these properties.
   DependentSizedArrayType *canonTy =
     DependentSizedArrayTypes.FindNodeOrInsertPos(ID, insertPos);
 
   // If we don't have one, build one.
   if (!canonTy) {
     canonTy = new (*this, TypeAlignment)
       DependentSizedArrayType(*this, QualType(canonElementType.Ty, 0),
                               QualType(), numElements, ASM, elementTypeQuals,
                               brackets);
     DependentSizedArrayTypes.InsertNode(canonTy, insertPos);
     Types.push_back(canonTy);
   }
 
   // Apply qualifiers from the element type to the array.
   QualType canon = getQualifiedType(QualType(canonTy,0),
                                     canonElementType.Quals);
 
   // If we didn't need extra canonicalization for the element type or the size
   // expression, then just use that as our result.
   if (QualType(canonElementType.Ty, 0) == elementType &&
       canonTy->getSizeExpr() == numElements)
     return canon;
 
   // Otherwise, we need to build a type which follows the spelling
   // of the element type.
   auto *sugaredType
     = new (*this, TypeAlignment)
         DependentSizedArrayType(*this, elementType, canon, numElements,
                                 ASM, elementTypeQuals, brackets);
   Types.push_back(sugaredType);
   return QualType(sugaredType, 0);
 }
 
 QualType ASTContext::getIncompleteArrayType(QualType elementType,
                                             ArrayType::ArraySizeModifier ASM,
                                             unsigned elementTypeQuals) const {
   llvm::FoldingSetNodeID ID;
   IncompleteArrayType::Profile(ID, elementType, ASM, elementTypeQuals);
 
   void *insertPos = nullptr;
   if (IncompleteArrayType *iat =
        IncompleteArrayTypes.FindNodeOrInsertPos(ID, insertPos))
     return QualType(iat, 0);
 
   // If the element type isn't canonical, this won't be a canonical type
   // either, so fill in the canonical type field.  We also have to pull
   // qualifiers off the element type.
   QualType canon;
 
   if (!elementType.isCanonical() || elementType.hasLocalQualifiers()) {
     SplitQualType canonSplit = getCanonicalType(elementType).split();
     canon = getIncompleteArrayType(QualType(canonSplit.Ty, 0),
                                    ASM, elementTypeQuals);
     canon = getQualifiedType(canon, canonSplit.Quals);
 
     // Get the new insert position for the node we care about.
     IncompleteArrayType *existing =
       IncompleteArrayTypes.FindNodeOrInsertPos(ID, insertPos);
     assert(!existing && "Shouldn't be in the map!"); (void) existing;
   }
 
   auto *newType = new (*this, TypeAlignment)
     IncompleteArrayType(elementType, canon, ASM, elementTypeQuals);
 
   IncompleteArrayTypes.InsertNode(newType, insertPos);
   Types.push_back(newType);
   return QualType(newType, 0);
 }
 
 ASTContext::BuiltinVectorTypeInfo
 ASTContext::getBuiltinVectorTypeInfo(const BuiltinType *Ty) const {
 #define SVE_INT_ELTTY(BITS, ELTS, SIGNED, NUMVECTORS)                          \
   {getIntTypeForBitwidth(BITS, SIGNED), llvm::ElementCount::getScalable(ELTS), \
    NUMVECTORS};
 
 #define SVE_ELTTY(ELTTY, ELTS, NUMVECTORS)                                     \
   {ELTTY, llvm::ElementCount::getScalable(ELTS), NUMVECTORS};
 
   switch (Ty->getKind()) {
   default:
     llvm_unreachable("Unsupported builtin vector type");
   case BuiltinType::SveInt8:
     return SVE_INT_ELTTY(8, 16, true, 1);
   case BuiltinType::SveUint8:
     return SVE_INT_ELTTY(8, 16, false, 1);
   case BuiltinType::SveInt8x2:
     return SVE_INT_ELTTY(8, 16, true, 2);
   case BuiltinType::SveUint8x2:
     return SVE_INT_ELTTY(8, 16, false, 2);
   case BuiltinType::SveInt8x3:
     return SVE_INT_ELTTY(8, 16, true, 3);
   case BuiltinType::SveUint8x3:
     return SVE_INT_ELTTY(8, 16, false, 3);
   case BuiltinType::SveInt8x4:
     return SVE_INT_ELTTY(8, 16, true, 4);
   case BuiltinType::SveUint8x4:
     return SVE_INT_ELTTY(8, 16, false, 4);
   case BuiltinType::SveInt16:
     return SVE_INT_ELTTY(16, 8, true, 1);
   case BuiltinType::SveUint16:
     return SVE_INT_ELTTY(16, 8, false, 1);
   case BuiltinType::SveInt16x2:
     return SVE_INT_ELTTY(16, 8, true, 2);
   case BuiltinType::SveUint16x2:
     return SVE_INT_ELTTY(16, 8, false, 2);
   case BuiltinType::SveInt16x3:
     return SVE_INT_ELTTY(16, 8, true, 3);
   case BuiltinType::SveUint16x3:
     return SVE_INT_ELTTY(16, 8, false, 3);
   case BuiltinType::SveInt16x4:
     return SVE_INT_ELTTY(16, 8, true, 4);
   case BuiltinType::SveUint16x4:
     return SVE_INT_ELTTY(16, 8, false, 4);
   case BuiltinType::SveInt32:
     return SVE_INT_ELTTY(32, 4, true, 1);
   case BuiltinType::SveUint32:
     return SVE_INT_ELTTY(32, 4, false, 1);
   case BuiltinType::SveInt32x2:
     return SVE_INT_ELTTY(32, 4, true, 2);
   case BuiltinType::SveUint32x2:
     return SVE_INT_ELTTY(32, 4, false, 2);
   case BuiltinType::SveInt32x3:
     return SVE_INT_ELTTY(32, 4, true, 3);
   case BuiltinType::SveUint32x3:
     return SVE_INT_ELTTY(32, 4, false, 3);
   case BuiltinType::SveInt32x4:
     return SVE_INT_ELTTY(32, 4, true, 4);
   case BuiltinType::SveUint32x4:
     return SVE_INT_ELTTY(32, 4, false, 4);
   case BuiltinType::SveInt64:
     return SVE_INT_ELTTY(64, 2, true, 1);
   case BuiltinType::SveUint64:
     return SVE_INT_ELTTY(64, 2, false, 1);
   case BuiltinType::SveInt64x2:
     return SVE_INT_ELTTY(64, 2, true, 2);
   case BuiltinType::SveUint64x2:
     return SVE_INT_ELTTY(64, 2, false, 2);
   case BuiltinType::SveInt64x3:
     return SVE_INT_ELTTY(64, 2, true, 3);
   case BuiltinType::SveUint64x3:
     return SVE_INT_ELTTY(64, 2, false, 3);
   case BuiltinType::SveInt64x4:
     return SVE_INT_ELTTY(64, 2, true, 4);
   case BuiltinType::SveUint64x4:
     return SVE_INT_ELTTY(64, 2, false, 4);
   case BuiltinType::SveBool:
     return SVE_ELTTY(BoolTy, 16, 1);
   case BuiltinType::SveFloat16:
     return SVE_ELTTY(HalfTy, 8, 1);
   case BuiltinType::SveFloat16x2:
     return SVE_ELTTY(HalfTy, 8, 2);
   case BuiltinType::SveFloat16x3:
     return SVE_ELTTY(HalfTy, 8, 3);
   case BuiltinType::SveFloat16x4:
     return SVE_ELTTY(HalfTy, 8, 4);
   case BuiltinType::SveFloat32:
     return SVE_ELTTY(FloatTy, 4, 1);
   case BuiltinType::SveFloat32x2:
     return SVE_ELTTY(FloatTy, 4, 2);
   case BuiltinType::SveFloat32x3:
     return SVE_ELTTY(FloatTy, 4, 3);
   case BuiltinType::SveFloat32x4:
     return SVE_ELTTY(FloatTy, 4, 4);
   case BuiltinType::SveFloat64:
     return SVE_ELTTY(DoubleTy, 2, 1);
   case BuiltinType::SveFloat64x2:
     return SVE_ELTTY(DoubleTy, 2, 2);
   case BuiltinType::SveFloat64x3:
     return SVE_ELTTY(DoubleTy, 2, 3);
   case BuiltinType::SveFloat64x4:
     return SVE_ELTTY(DoubleTy, 2, 4);
   case BuiltinType::SveBFloat16:
     return SVE_ELTTY(BFloat16Ty, 8, 1);
   case BuiltinType::SveBFloat16x2:
     return SVE_ELTTY(BFloat16Ty, 8, 2);
   case BuiltinType::SveBFloat16x3:
     return SVE_ELTTY(BFloat16Ty, 8, 3);
   case BuiltinType::SveBFloat16x4:
     return SVE_ELTTY(BFloat16Ty, 8, 4);
 #define RVV_VECTOR_TYPE_INT(Name, Id, SingletonId, NumEls, ElBits, NF,         \
                             IsSigned)                                          \
   case BuiltinType::Id:                                                        \
     return {getIntTypeForBitwidth(ElBits, IsSigned),                           \
             llvm::ElementCount::getScalable(NumEls), NF};
 #define RVV_VECTOR_TYPE_FLOAT(Name, Id, SingletonId, NumEls, ElBits, NF)       \
   case BuiltinType::Id:                                                        \
     return {ElBits == 16 ? Float16Ty : (ElBits == 32 ? FloatTy : DoubleTy),    \
             llvm::ElementCount::getScalable(NumEls), NF};
 #define RVV_PREDICATE_TYPE(Name, Id, SingletonId, NumEls)                      \
   case BuiltinType::Id:                                                        \
     return {BoolTy, llvm::ElementCount::getScalable(NumEls), 1};
 #include "clang/Basic/RISCVVTypes.def"
   }
 }
 
 /// getScalableVectorType - Return the unique reference to a scalable vector
 /// type of the specified element type and size. VectorType must be a built-in
 /// type.
 QualType ASTContext::getScalableVectorType(QualType EltTy,
                                            unsigned NumElts) const {
   if (Target->hasAArch64SVETypes()) {
     uint64_t EltTySize = getTypeSize(EltTy);
 #define SVE_VECTOR_TYPE(Name, MangledName, Id, SingletonId, NumEls, ElBits,    \
                         IsSigned, IsFP, IsBF)                                  \
   if (!EltTy->isBooleanType() &&                                               \
       ((EltTy->hasIntegerRepresentation() &&                                   \
         EltTy->hasSignedIntegerRepresentation() == IsSigned) ||                \
        (EltTy->hasFloatingRepresentation() && !EltTy->isBFloat16Type() &&      \
         IsFP && !IsBF) ||                                                      \
        (EltTy->hasFloatingRepresentation() && EltTy->isBFloat16Type() &&       \
         IsBF && !IsFP)) &&                                                     \
       EltTySize == ElBits && NumElts == NumEls) {                              \
     return SingletonId;                                                        \
   }
 #define SVE_PREDICATE_TYPE(Name, MangledName, Id, SingletonId, NumEls)         \
   if (EltTy->isBooleanType() && NumElts == NumEls)                             \
     return SingletonId;
 #include "clang/Basic/AArch64SVEACLETypes.def"
   } else if (Target->hasRISCVVTypes()) {
     uint64_t EltTySize = getTypeSize(EltTy);
 #define RVV_VECTOR_TYPE(Name, Id, SingletonId, NumEls, ElBits, NF, IsSigned,   \
                         IsFP)                                                  \
     if (!EltTy->isBooleanType() &&                                             \
         ((EltTy->hasIntegerRepresentation() &&                                 \
           EltTy->hasSignedIntegerRepresentation() == IsSigned) ||              \
          (EltTy->hasFloatingRepresentation() && IsFP)) &&                      \
         EltTySize == ElBits && NumElts == NumEls)                              \
       return SingletonId;
 #define RVV_PREDICATE_TYPE(Name, Id, SingletonId, NumEls)                      \
     if (EltTy->isBooleanType() && NumElts == NumEls)                           \
       return SingletonId;
 #include "clang/Basic/RISCVVTypes.def"
   }
   return QualType();
 }
 
 /// getVectorType - Return the unique reference to a vector type of
 /// the specified element type and size. VectorType must be a built-in type.
 QualType ASTContext::getVectorType(QualType vecType, unsigned NumElts,
                                    VectorType::VectorKind VecKind) const {
   assert(vecType->isBuiltinType());
 
   // Check if we've already instantiated a vector of this type.
   llvm::FoldingSetNodeID ID;
   VectorType::Profile(ID, vecType, NumElts, Type::Vector, VecKind);
 
   void *InsertPos = nullptr;
   if (VectorType *VTP = VectorTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(VTP, 0);
 
   // If the element type isn't canonical, this won't be a canonical type either,
   // so fill in the canonical type field.
   QualType Canonical;
   if (!vecType.isCanonical()) {
     Canonical = getVectorType(getCanonicalType(vecType), NumElts, VecKind);
 
     // Get the new insert position for the node we care about.
     VectorType *NewIP = VectorTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
   auto *New = new (*this, TypeAlignment)
     VectorType(vecType, NumElts, Canonical, VecKind);
   VectorTypes.InsertNode(New, InsertPos);
   Types.push_back(New);
   return QualType(New, 0);
 }
 
 QualType
 ASTContext::getDependentVectorType(QualType VecType, Expr *SizeExpr,
                                    SourceLocation AttrLoc,
                                    VectorType::VectorKind VecKind) const {
   llvm::FoldingSetNodeID ID;
   DependentVectorType::Profile(ID, *this, getCanonicalType(VecType), SizeExpr,
                                VecKind);
   void *InsertPos = nullptr;
   DependentVectorType *Canon =
       DependentVectorTypes.FindNodeOrInsertPos(ID, InsertPos);
   DependentVectorType *New;
 
   if (Canon) {
     New = new (*this, TypeAlignment) DependentVectorType(
         *this, VecType, QualType(Canon, 0), SizeExpr, AttrLoc, VecKind);
   } else {
     QualType CanonVecTy = getCanonicalType(VecType);
     if (CanonVecTy == VecType) {
       New = new (*this, TypeAlignment) DependentVectorType(
           *this, VecType, QualType(), SizeExpr, AttrLoc, VecKind);
 
       DependentVectorType *CanonCheck =
           DependentVectorTypes.FindNodeOrInsertPos(ID, InsertPos);
       assert(!CanonCheck &&
              "Dependent-sized vector_size canonical type broken");
       (void)CanonCheck;
       DependentVectorTypes.InsertNode(New, InsertPos);
     } else {
       QualType CanonTy = getDependentVectorType(CanonVecTy, SizeExpr,
                                                 SourceLocation(), VecKind);
       New = new (*this, TypeAlignment) DependentVectorType(
           *this, VecType, CanonTy, SizeExpr, AttrLoc, VecKind);
     }
   }
 
   Types.push_back(New);
   return QualType(New, 0);
 }
 
 /// getExtVectorType - Return the unique reference to an extended vector type of
 /// the specified element type and size. VectorType must be a built-in type.
 QualType
 ASTContext::getExtVectorType(QualType vecType, unsigned NumElts) const {
   assert(vecType->isBuiltinType() || vecType->isDependentType());
 
   // Check if we've already instantiated a vector of this type.
   llvm::FoldingSetNodeID ID;
   VectorType::Profile(ID, vecType, NumElts, Type::ExtVector,
                       VectorType::GenericVector);
   void *InsertPos = nullptr;
   if (VectorType *VTP = VectorTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(VTP, 0);
 
   // If the element type isn't canonical, this won't be a canonical type either,
   // so fill in the canonical type field.
   QualType Canonical;
   if (!vecType.isCanonical()) {
     Canonical = getExtVectorType(getCanonicalType(vecType), NumElts);
 
     // Get the new insert position for the node we care about.
     VectorType *NewIP = VectorTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
   auto *New = new (*this, TypeAlignment)
     ExtVectorType(vecType, NumElts, Canonical);
   VectorTypes.InsertNode(New, InsertPos);
   Types.push_back(New);
   return QualType(New, 0);
 }
 
 QualType
 ASTContext::getDependentSizedExtVectorType(QualType vecType,
                                            Expr *SizeExpr,
                                            SourceLocation AttrLoc) const {
   llvm::FoldingSetNodeID ID;
   DependentSizedExtVectorType::Profile(ID, *this, getCanonicalType(vecType),
                                        SizeExpr);
 
   void *InsertPos = nullptr;
   DependentSizedExtVectorType *Canon
     = DependentSizedExtVectorTypes.FindNodeOrInsertPos(ID, InsertPos);
   DependentSizedExtVectorType *New;
   if (Canon) {
     // We already have a canonical version of this array type; use it as
     // the canonical type for a newly-built type.
     New = new (*this, TypeAlignment)
       DependentSizedExtVectorType(*this, vecType, QualType(Canon, 0),
                                   SizeExpr, AttrLoc);
   } else {
     QualType CanonVecTy = getCanonicalType(vecType);
     if (CanonVecTy == vecType) {
       New = new (*this, TypeAlignment)
         DependentSizedExtVectorType(*this, vecType, QualType(), SizeExpr,
                                     AttrLoc);
 
       DependentSizedExtVectorType *CanonCheck
         = DependentSizedExtVectorTypes.FindNodeOrInsertPos(ID, InsertPos);
       assert(!CanonCheck && "Dependent-sized ext_vector canonical type broken");
       (void)CanonCheck;
       DependentSizedExtVectorTypes.InsertNode(New, InsertPos);
     } else {
       QualType CanonExtTy = getDependentSizedExtVectorType(CanonVecTy, SizeExpr,
                                                            SourceLocation());
       New = new (*this, TypeAlignment) DependentSizedExtVectorType(
           *this, vecType, CanonExtTy, SizeExpr, AttrLoc);
     }
   }
 
   Types.push_back(New);
   return QualType(New, 0);
 }
 
 QualType ASTContext::getConstantMatrixType(QualType ElementTy, unsigned NumRows,
                                            unsigned NumColumns) const {
   llvm::FoldingSetNodeID ID;
   ConstantMatrixType::Profile(ID, ElementTy, NumRows, NumColumns,
                               Type::ConstantMatrix);
 
   assert(MatrixType::isValidElementType(ElementTy) &&
          "need a valid element type");
   assert(ConstantMatrixType::isDimensionValid(NumRows) &&
          ConstantMatrixType::isDimensionValid(NumColumns) &&
          "need valid matrix dimensions");
   void *InsertPos = nullptr;
   if (ConstantMatrixType *MTP = MatrixTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(MTP, 0);
 
   QualType Canonical;
   if (!ElementTy.isCanonical()) {
     Canonical =
         getConstantMatrixType(getCanonicalType(ElementTy), NumRows, NumColumns);
 
     ConstantMatrixType *NewIP = MatrixTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Matrix type shouldn't already exist in the map");
     (void)NewIP;
   }
 
   auto *New = new (*this, TypeAlignment)
       ConstantMatrixType(ElementTy, NumRows, NumColumns, Canonical);
   MatrixTypes.InsertNode(New, InsertPos);
   Types.push_back(New);
   return QualType(New, 0);
 }
 
 QualType ASTContext::getDependentSizedMatrixType(QualType ElementTy,
                                                  Expr *RowExpr,
                                                  Expr *ColumnExpr,
                                                  SourceLocation AttrLoc) const {
   QualType CanonElementTy = getCanonicalType(ElementTy);
   llvm::FoldingSetNodeID ID;
   DependentSizedMatrixType::Profile(ID, *this, CanonElementTy, RowExpr,
                                     ColumnExpr);
 
   void *InsertPos = nullptr;
   DependentSizedMatrixType *Canon =
       DependentSizedMatrixTypes.FindNodeOrInsertPos(ID, InsertPos);
 
   if (!Canon) {
     Canon = new (*this, TypeAlignment) DependentSizedMatrixType(
         *this, CanonElementTy, QualType(), RowExpr, ColumnExpr, AttrLoc);
 #ifndef NDEBUG
     DependentSizedMatrixType *CanonCheck =
         DependentSizedMatrixTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!CanonCheck && "Dependent-sized matrix canonical type broken");
 #endif
     DependentSizedMatrixTypes.InsertNode(Canon, InsertPos);
     Types.push_back(Canon);
   }
 
   // Already have a canonical version of the matrix type
   //
   // If it exactly matches the requested type, use it directly.
   if (Canon->getElementType() == ElementTy && Canon->getRowExpr() == RowExpr &&
       Canon->getRowExpr() == ColumnExpr)
     return QualType(Canon, 0);
 
   // Use Canon as the canonical type for newly-built type.
   DependentSizedMatrixType *New = new (*this, TypeAlignment)
       DependentSizedMatrixType(*this, ElementTy, QualType(Canon, 0), RowExpr,
                                ColumnExpr, AttrLoc);
   Types.push_back(New);
   return QualType(New, 0);
 }
 
 QualType ASTContext::getDependentAddressSpaceType(QualType PointeeType,
                                                   Expr *AddrSpaceExpr,
                                                   SourceLocation AttrLoc) const {
   assert(AddrSpaceExpr->isInstantiationDependent());
 
   QualType canonPointeeType = getCanonicalType(PointeeType);
 
   void *insertPos = nullptr;
   llvm::FoldingSetNodeID ID;
   DependentAddressSpaceType::Profile(ID, *this, canonPointeeType,
                                      AddrSpaceExpr);
 
   DependentAddressSpaceType *canonTy =
     DependentAddressSpaceTypes.FindNodeOrInsertPos(ID, insertPos);
 
   if (!canonTy) {
     canonTy = new (*this, TypeAlignment)
       DependentAddressSpaceType(*this, canonPointeeType,
                                 QualType(), AddrSpaceExpr, AttrLoc);
     DependentAddressSpaceTypes.InsertNode(canonTy, insertPos);
     Types.push_back(canonTy);
   }
 
   if (canonPointeeType == PointeeType &&
       canonTy->getAddrSpaceExpr() == AddrSpaceExpr)
     return QualType(canonTy, 0);
 
   auto *sugaredType
     = new (*this, TypeAlignment)
         DependentAddressSpaceType(*this, PointeeType, QualType(canonTy, 0),
                                   AddrSpaceExpr, AttrLoc);
   Types.push_back(sugaredType);
   return QualType(sugaredType, 0);
 }
 
 /// Determine whether \p T is canonical as the result type of a function.
 static bool isCanonicalResultType(QualType T) {
   return T.isCanonical() &&
          (T.getObjCLifetime() == Qualifiers::OCL_None ||
           T.getObjCLifetime() == Qualifiers::OCL_ExplicitNone);
 }
 
 /// getFunctionNoProtoType - Return a K&R style C function type like 'int()'.
 QualType
 ASTContext::getFunctionNoProtoType(QualType ResultTy,
                                    const FunctionType::ExtInfo &Info) const {
   // Unique functions, to guarantee there is only one function of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
   FunctionNoProtoType::Profile(ID, ResultTy, Info);
 
   void *InsertPos = nullptr;
   if (FunctionNoProtoType *FT =
         FunctionNoProtoTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(FT, 0);
 
   QualType Canonical;
   if (!isCanonicalResultType(ResultTy)) {
     Canonical =
       getFunctionNoProtoType(getCanonicalFunctionResultType(ResultTy), Info);
 
     // Get the new insert position for the node we care about.
     FunctionNoProtoType *NewIP =
       FunctionNoProtoTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
 
   auto *New = new (*this, TypeAlignment)
     FunctionNoProtoType(ResultTy, Canonical, Info);
   Types.push_back(New);
   FunctionNoProtoTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
 }
 
 CanQualType
 ASTContext::getCanonicalFunctionResultType(QualType ResultType) const {
   CanQualType CanResultType = getCanonicalType(ResultType);
 
   // Canonical result types do not have ARC lifetime qualifiers.
   if (CanResultType.getQualifiers().hasObjCLifetime()) {
     Qualifiers Qs = CanResultType.getQualifiers();
     Qs.removeObjCLifetime();
     return CanQualType::CreateUnsafe(
              getQualifiedType(CanResultType.getUnqualifiedType(), Qs));
   }
 
   return CanResultType;
 }
 
 static bool isCanonicalExceptionSpecification(
     const FunctionProtoType::ExceptionSpecInfo &ESI, bool NoexceptInType) {
   if (ESI.Type == EST_None)
     return true;
   if (!NoexceptInType)
     return false;
 
   // C++17 onwards: exception specification is part of the type, as a simple
   // boolean "can this function type throw".
   if (ESI.Type == EST_BasicNoexcept)
     return true;
 
   // A noexcept(expr) specification is (possibly) canonical if expr is
   // value-dependent.
   if (ESI.Type == EST_DependentNoexcept)
     return true;
 
   // A dynamic exception specification is canonical if it only contains pack
   // expansions (so we can't tell whether it's non-throwing) and all its
   // contained types are canonical.
   if (ESI.Type == EST_Dynamic) {
     bool AnyPackExpansions = false;
     for (QualType ET : ESI.Exceptions) {
       if (!ET.isCanonical())
         return false;
       if (ET->getAs<PackExpansionType>())
         AnyPackExpansions = true;
     }
     return AnyPackExpansions;
   }
 
   return false;
 }
 
 QualType ASTContext::getFunctionTypeInternal(
     QualType ResultTy, ArrayRef<QualType> ArgArray,
     const FunctionProtoType::ExtProtoInfo &EPI, bool OnlyWantCanonical) const {
   size_t NumArgs = ArgArray.size();
 
   // Unique functions, to guarantee there is only one function of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
   FunctionProtoType::Profile(ID, ResultTy, ArgArray.begin(), NumArgs, EPI,
                              *this, true);
 
   QualType Canonical;
   bool Unique = false;
 
   void *InsertPos = nullptr;
   if (FunctionProtoType *FPT =
         FunctionProtoTypes.FindNodeOrInsertPos(ID, InsertPos)) {
     QualType Existing = QualType(FPT, 0);
 
     // If we find a pre-existing equivalent FunctionProtoType, we can just reuse
     // it so long as our exception specification doesn't contain a dependent
     // noexcept expression, or we're just looking for a canonical type.
     // Otherwise, we're going to need to create a type
     // sugar node to hold the concrete expression.
     if (OnlyWantCanonical || !isComputedNoexcept(EPI.ExceptionSpec.Type) ||
         EPI.ExceptionSpec.NoexceptExpr == FPT->getNoexceptExpr())
       return Existing;
 
     // We need a new type sugar node for this one, to hold the new noexcept
     // expression. We do no canonicalization here, but that's OK since we don't
     // expect to see the same noexcept expression much more than once.
     Canonical = getCanonicalType(Existing);
     Unique = true;
   }
 
   bool NoexceptInType = getLangOpts().CPlusPlus17;
   bool IsCanonicalExceptionSpec =
       isCanonicalExceptionSpecification(EPI.ExceptionSpec, NoexceptInType);
 
   // Determine whether the type being created is already canonical or not.
   bool isCanonical = !Unique && IsCanonicalExceptionSpec &&
                      isCanonicalResultType(ResultTy) && !EPI.HasTrailingReturn;
   for (unsigned i = 0; i != NumArgs && isCanonical; ++i)
     if (!ArgArray[i].isCanonicalAsParam())
       isCanonical = false;
 
   if (OnlyWantCanonical)
     assert(isCanonical &&
            "given non-canonical parameters constructing canonical type");
 
   // If this type isn't canonical, get the canonical version of it if we don't
   // already have it. The exception spec is only partially part of the
   // canonical type, and only in C++17 onwards.
   if (!isCanonical && Canonical.isNull()) {
     SmallVector<QualType, 16> CanonicalArgs;
     CanonicalArgs.reserve(NumArgs);
     for (unsigned i = 0; i != NumArgs; ++i)
       CanonicalArgs.push_back(getCanonicalParamType(ArgArray[i]));
 
     llvm::SmallVector<QualType, 8> ExceptionTypeStorage;
     FunctionProtoType::ExtProtoInfo CanonicalEPI = EPI;
     CanonicalEPI.HasTrailingReturn = false;
 
     if (IsCanonicalExceptionSpec) {
       // Exception spec is already OK.
     } else if (NoexceptInType) {
       switch (EPI.ExceptionSpec.Type) {
       case EST_Unparsed: case EST_Unevaluated: case EST_Uninstantiated:
         // We don't know yet. It shouldn't matter what we pick here; no-one
         // should ever look at this.
         LLVM_FALLTHROUGH;
       case EST_None: case EST_MSAny: case EST_NoexceptFalse:
         CanonicalEPI.ExceptionSpec.Type = EST_None;
         break;
 
         // A dynamic exception specification is almost always "not noexcept",
         // with the exception that a pack expansion might expand to no types.
       case EST_Dynamic: {
         bool AnyPacks = false;
         for (QualType ET : EPI.ExceptionSpec.Exceptions) {
           if (ET->getAs<PackExpansionType>())
             AnyPacks = true;
           ExceptionTypeStorage.push_back(getCanonicalType(ET));
         }
         if (!AnyPacks)
           CanonicalEPI.ExceptionSpec.Type = EST_None;
         else {
           CanonicalEPI.ExceptionSpec.Type = EST_Dynamic;
           CanonicalEPI.ExceptionSpec.Exceptions = ExceptionTypeStorage;
         }
         break;
       }
 
       case EST_DynamicNone:
       case EST_BasicNoexcept:
       case EST_NoexceptTrue:
       case EST_NoThrow:
         CanonicalEPI.ExceptionSpec.Type = EST_BasicNoexcept;
         break;
 
       case EST_DependentNoexcept:
         llvm_unreachable("dependent noexcept is already canonical");
       }
     } else {
       CanonicalEPI.ExceptionSpec = FunctionProtoType::ExceptionSpecInfo();
     }
 
     // Adjust the canonical function result type.
     CanQualType CanResultTy = getCanonicalFunctionResultType(ResultTy);
     Canonical =
         getFunctionTypeInternal(CanResultTy, CanonicalArgs, CanonicalEPI, true);
 
     // Get the new insert position for the node we care about.
     FunctionProtoType *NewIP =
       FunctionProtoTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
 
   // Compute the needed size to hold this FunctionProtoType and the
   // various trailing objects.
   auto ESH = FunctionProtoType::getExceptionSpecSize(
       EPI.ExceptionSpec.Type, EPI.ExceptionSpec.Exceptions.size());
   size_t Size = FunctionProtoType::totalSizeToAlloc<
       QualType, SourceLocation, FunctionType::FunctionTypeExtraBitfields,
       FunctionType::ExceptionType, Expr *, FunctionDecl *,
       FunctionProtoType::ExtParameterInfo, Qualifiers>(
       NumArgs, EPI.Variadic,
       FunctionProtoType::hasExtraBitfields(EPI.ExceptionSpec.Type),
       ESH.NumExceptionType, ESH.NumExprPtr, ESH.NumFunctionDeclPtr,
       EPI.ExtParameterInfos ? NumArgs : 0,
       EPI.TypeQuals.hasNonFastQualifiers() ? 1 : 0);
 
   auto *FTP = (FunctionProtoType *)Allocate(Size, TypeAlignment);
   FunctionProtoType::ExtProtoInfo newEPI = EPI;
   new (FTP) FunctionProtoType(ResultTy, ArgArray, Canonical, newEPI);
   Types.push_back(FTP);
   if (!Unique)
     FunctionProtoTypes.InsertNode(FTP, InsertPos);
   return QualType(FTP, 0);
 }
 
 QualType ASTContext::getPipeType(QualType T, bool ReadOnly) const {
   llvm::FoldingSetNodeID ID;
   PipeType::Profile(ID, T, ReadOnly);
 
   void *InsertPos = nullptr;
   if (PipeType *PT = PipeTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(PT, 0);
 
   // If the pipe element type isn't canonical, this won't be a canonical type
   // either, so fill in the canonical type field.
   QualType Canonical;
   if (!T.isCanonical()) {
     Canonical = getPipeType(getCanonicalType(T), ReadOnly);
 
     // Get the new insert position for the node we care about.
     PipeType *NewIP = PipeTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!");
     (void)NewIP;
   }
   auto *New = new (*this, TypeAlignment) PipeType(T, Canonical, ReadOnly);
   Types.push_back(New);
   PipeTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
 }
 
 QualType ASTContext::adjustStringLiteralBaseType(QualType Ty) const {
   // OpenCL v1.1 s6.5.3: a string literal is in the constant address space.
   return LangOpts.OpenCL ? getAddrSpaceQualType(Ty, LangAS::opencl_constant)
                          : Ty;
 }
 
 QualType ASTContext::getReadPipeType(QualType T) const {
   return getPipeType(T, true);
 }
 
 QualType ASTContext::getWritePipeType(QualType T) const {
   return getPipeType(T, false);
 }
 
 QualType ASTContext::getBitIntType(bool IsUnsigned, unsigned NumBits) const {
   llvm::FoldingSetNodeID ID;
   BitIntType::Profile(ID, IsUnsigned, NumBits);
 
   void *InsertPos = nullptr;
   if (BitIntType *EIT = BitIntTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(EIT, 0);
 
   auto *New = new (*this, TypeAlignment) BitIntType(IsUnsigned, NumBits);
   BitIntTypes.InsertNode(New, InsertPos);
   Types.push_back(New);
   return QualType(New, 0);
 }
 
 QualType ASTContext::getDependentBitIntType(bool IsUnsigned,
                                             Expr *NumBitsExpr) const {
   assert(NumBitsExpr->isInstantiationDependent() && "Only good for dependent");
   llvm::FoldingSetNodeID ID;
   DependentBitIntType::Profile(ID, *this, IsUnsigned, NumBitsExpr);
 
   void *InsertPos = nullptr;
   if (DependentBitIntType *Existing =
           DependentBitIntTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(Existing, 0);
 
   auto *New = new (*this, TypeAlignment)
       DependentBitIntType(*this, IsUnsigned, NumBitsExpr);
   DependentBitIntTypes.InsertNode(New, InsertPos);
 
   Types.push_back(New);
   return QualType(New, 0);
 }
 
 #ifndef NDEBUG
 static bool NeedsInjectedClassNameType(const RecordDecl *D) {
   if (!isa<CXXRecordDecl>(D)) return false;
   const auto *RD = cast<CXXRecordDecl>(D);
   if (isa<ClassTemplatePartialSpecializationDecl>(RD))
     return true;
   if (RD->getDescribedClassTemplate() &&
       !isa<ClassTemplateSpecializationDecl>(RD))
     return true;
   return false;
 }
 #endif
 
 /// getInjectedClassNameType - Return the unique reference to the
 /// injected class name type for the specified templated declaration.
 QualType ASTContext::getInjectedClassNameType(CXXRecordDecl *Decl,
                                               QualType TST) const {
   assert(NeedsInjectedClassNameType(Decl));
   if (Decl->TypeForDecl) {
     assert(isa<InjectedClassNameType>(Decl->TypeForDecl));
   } else if (CXXRecordDecl *PrevDecl = Decl->getPreviousDecl()) {
     assert(PrevDecl->TypeForDecl && "previous declaration has no type");
     Decl->TypeForDecl = PrevDecl->TypeForDecl;
     assert(isa<InjectedClassNameType>(Decl->TypeForDecl));
   } else {
     Type *newType =
       new (*this, TypeAlignment) InjectedClassNameType(Decl, TST);
     Decl->TypeForDecl = newType;
     Types.push_back(newType);
   }
   return QualType(Decl->TypeForDecl, 0);
 }
 
 /// getTypeDeclType - Return the unique reference to the type for the
 /// specified type declaration.
 QualType ASTContext::getTypeDeclTypeSlow(const TypeDecl *Decl) const {
   assert(Decl && "Passed null for Decl param");
   assert(!Decl->TypeForDecl && "TypeForDecl present in slow case");
 
   if (const auto *Typedef = dyn_cast<TypedefNameDecl>(Decl))
     return getTypedefType(Typedef);
 
   assert(!isa<TemplateTypeParmDecl>(Decl) &&
          "Template type parameter types are always available.");
 
   if (const auto *Record = dyn_cast<RecordDecl>(Decl)) {
     assert(Record->isFirstDecl() && "struct/union has previous declaration");
     assert(!NeedsInjectedClassNameType(Record));
     return getRecordType(Record);
   } else if (const auto *Enum = dyn_cast<EnumDecl>(Decl)) {
     assert(Enum->isFirstDecl() && "enum has previous declaration");
     return getEnumType(Enum);
   } else if (const auto *Using = dyn_cast<UnresolvedUsingTypenameDecl>(Decl)) {
     return getUnresolvedUsingType(Using);
   } else
     llvm_unreachable("TypeDecl without a type?");
 
   return QualType(Decl->TypeForDecl, 0);
 }
 
 /// getTypedefType - Return the unique reference to the type for the
 /// specified typedef name decl.
 QualType ASTContext::getTypedefType(const TypedefNameDecl *Decl,
                                     QualType Underlying) const {
   if (Decl->TypeForDecl) return QualType(Decl->TypeForDecl, 0);
 
   if (Underlying.isNull())
     Underlying = Decl->getUnderlyingType();
   QualType Canonical = getCanonicalType(Underlying);
   auto *newType = new (*this, TypeAlignment)
       TypedefType(Type::Typedef, Decl, Underlying, Canonical);
   Decl->TypeForDecl = newType;
   Types.push_back(newType);
   return QualType(newType, 0);
 }
 
 QualType ASTContext::getUsingType(const UsingShadowDecl *Found,
                                   QualType Underlying) const {
   llvm::FoldingSetNodeID ID;
   UsingType::Profile(ID, Found);
 
   void *InsertPos = nullptr;
   UsingType *T = UsingTypes.FindNodeOrInsertPos(ID, InsertPos);
   if (T)
     return QualType(T, 0);
 
   assert(!Underlying.hasLocalQualifiers());
   assert(Underlying == getTypeDeclType(cast<TypeDecl>(Found->getTargetDecl())));
   QualType Canon = Underlying.getCanonicalType();
 
   UsingType *NewType =
       new (*this, TypeAlignment) UsingType(Found, Underlying, Canon);
   Types.push_back(NewType);
   UsingTypes.InsertNode(NewType, InsertPos);
   return QualType(NewType, 0);
 }
 
 QualType ASTContext::getRecordType(const RecordDecl *Decl) const {
   if (Decl->TypeForDecl) return QualType(Decl->TypeForDecl, 0);
 
   if (const RecordDecl *PrevDecl = Decl->getPreviousDecl())
     if (PrevDecl->TypeForDecl)
       return QualType(Decl->TypeForDecl = PrevDecl->TypeForDecl, 0);
 
   auto *newType = new (*this, TypeAlignment) RecordType(Decl);
   Decl->TypeForDecl = newType;
   Types.push_back(newType);
   return QualType(newType, 0);
 }
 
 QualType ASTContext::getEnumType(const EnumDecl *Decl) const {
   if (Decl->TypeForDecl) return QualType(Decl->TypeForDecl, 0);
 
   if (const EnumDecl *PrevDecl = Decl->getPreviousDecl())
     if (PrevDecl->TypeForDecl)
       return QualType(Decl->TypeForDecl = PrevDecl->TypeForDecl, 0);
 
   auto *newType = new (*this, TypeAlignment) EnumType(Decl);
   Decl->TypeForDecl = newType;
   Types.push_back(newType);
   return QualType(newType, 0);
 }
 
 QualType ASTContext::getUnresolvedUsingType(
     const UnresolvedUsingTypenameDecl *Decl) const {
   if (Decl->TypeForDecl)
     return QualType(Decl->TypeForDecl, 0);
 
   if (const UnresolvedUsingTypenameDecl *CanonicalDecl =
           Decl->getCanonicalDecl())
     if (CanonicalDecl->TypeForDecl)
       return QualType(Decl->TypeForDecl = CanonicalDecl->TypeForDecl, 0);
 
   Type *newType = new (*this, TypeAlignment) UnresolvedUsingType(Decl);
   Decl->TypeForDecl = newType;
   Types.push_back(newType);
   return QualType(newType, 0);
 }
 
 QualType ASTContext::getAttributedType(attr::Kind attrKind,
                                        QualType modifiedType,
                                        QualType equivalentType) {
   llvm::FoldingSetNodeID id;
   AttributedType::Profile(id, attrKind, modifiedType, equivalentType);
 
   void *insertPos = nullptr;
   AttributedType *type = AttributedTypes.FindNodeOrInsertPos(id, insertPos);
   if (type) return QualType(type, 0);
 
   QualType canon = getCanonicalType(equivalentType);
   type = new (*this, TypeAlignment)
       AttributedType(canon, attrKind, modifiedType, equivalentType);
 
   Types.push_back(type);
   AttributedTypes.InsertNode(type, insertPos);
 
   return QualType(type, 0);
 }
 
 /// Retrieve a substitution-result type.
 QualType
 ASTContext::getSubstTemplateTypeParmType(const TemplateTypeParmType *Parm,
                                          QualType Replacement) const {
   assert(Replacement.isCanonical()
          && "replacement types must always be canonical");
 
   llvm::FoldingSetNodeID ID;
   SubstTemplateTypeParmType::Profile(ID, Parm, Replacement);
   void *InsertPos = nullptr;
   SubstTemplateTypeParmType *SubstParm
     = SubstTemplateTypeParmTypes.FindNodeOrInsertPos(ID, InsertPos);
 
   if (!SubstParm) {
     SubstParm = new (*this, TypeAlignment)
       SubstTemplateTypeParmType(Parm, Replacement);
     Types.push_back(SubstParm);
     SubstTemplateTypeParmTypes.InsertNode(SubstParm, InsertPos);
   }
 
   return QualType(SubstParm, 0);
 }
 
 /// Retrieve a
 QualType ASTContext::getSubstTemplateTypeParmPackType(
                                           const TemplateTypeParmType *Parm,
                                               const TemplateArgument &ArgPack) {
 #ifndef NDEBUG
   for (const auto &P : ArgPack.pack_elements()) {
     assert(P.getKind() == TemplateArgument::Type &&"Pack contains a non-type");
     assert(P.getAsType().isCanonical() && "Pack contains non-canonical type");
   }
 #endif
 
   llvm::FoldingSetNodeID ID;
   SubstTemplateTypeParmPackType::Profile(ID, Parm, ArgPack);
   void *InsertPos = nullptr;
   if (SubstTemplateTypeParmPackType *SubstParm
         = SubstTemplateTypeParmPackTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(SubstParm, 0);
 
   QualType Canon;
   if (!Parm->isCanonicalUnqualified()) {
     Canon = getCanonicalType(QualType(Parm, 0));
     Canon = getSubstTemplateTypeParmPackType(cast<TemplateTypeParmType>(Canon),
                                              ArgPack);
     SubstTemplateTypeParmPackTypes.FindNodeOrInsertPos(ID, InsertPos);
   }
 
   auto *SubstParm
     = new (*this, TypeAlignment) SubstTemplateTypeParmPackType(Parm, Canon,
                                                                ArgPack);
   Types.push_back(SubstParm);
   SubstTemplateTypeParmPackTypes.InsertNode(SubstParm, InsertPos);
   return QualType(SubstParm, 0);
 }
 
 /// Retrieve the template type parameter type for a template
 /// parameter or parameter pack with the given depth, index, and (optionally)
 /// name.
 QualType ASTContext::getTemplateTypeParmType(unsigned Depth, unsigned Index,
                                              bool ParameterPack,
                                              TemplateTypeParmDecl *TTPDecl) const {
   llvm::FoldingSetNodeID ID;
   TemplateTypeParmType::Profile(ID, Depth, Index, ParameterPack, TTPDecl);
   void *InsertPos = nullptr;
   TemplateTypeParmType *TypeParm
     = TemplateTypeParmTypes.FindNodeOrInsertPos(ID, InsertPos);
 
   if (TypeParm)
     return QualType(TypeParm, 0);
 
   if (TTPDecl) {
     QualType Canon = getTemplateTypeParmType(Depth, Index, ParameterPack);
     TypeParm = new (*this, TypeAlignment) TemplateTypeParmType(TTPDecl, Canon);
 
     TemplateTypeParmType *TypeCheck
       = TemplateTypeParmTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!TypeCheck && "Template type parameter canonical type broken");
     (void)TypeCheck;
   } else
     TypeParm = new (*this, TypeAlignment)
       TemplateTypeParmType(Depth, Index, ParameterPack);
 
   Types.push_back(TypeParm);
   TemplateTypeParmTypes.InsertNode(TypeParm, InsertPos);
 
   return QualType(TypeParm, 0);
 }
 
 TypeSourceInfo *
 ASTContext::getTemplateSpecializationTypeInfo(TemplateName Name,
                                               SourceLocation NameLoc,
                                         const TemplateArgumentListInfo &Args,
                                               QualType Underlying) const {
   assert(!Name.getAsDependentTemplateName() &&
          "No dependent template names here!");
   QualType TST = getTemplateSpecializationType(Name, Args, Underlying);
 
   TypeSourceInfo *DI = CreateTypeSourceInfo(TST);
   TemplateSpecializationTypeLoc TL =
       DI->getTypeLoc().castAs<TemplateSpecializationTypeLoc>();
   TL.setTemplateKeywordLoc(SourceLocation());
   TL.setTemplateNameLoc(NameLoc);
   TL.setLAngleLoc(Args.getLAngleLoc());
   TL.setRAngleLoc(Args.getRAngleLoc());
   for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i)
     TL.setArgLocInfo(i, Args[i].getLocInfo());
   return DI;
 }
 
 QualType
 ASTContext::getTemplateSpecializationType(TemplateName Template,
                                           const TemplateArgumentListInfo &Args,
                                           QualType Underlying) const {
   assert(!Template.getAsDependentTemplateName() &&
          "No dependent template names here!");
 
   SmallVector<TemplateArgument, 4> ArgVec;
   ArgVec.reserve(Args.size());
   for (const TemplateArgumentLoc &Arg : Args.arguments())
     ArgVec.push_back(Arg.getArgument());
 
   return getTemplateSpecializationType(Template, ArgVec, Underlying);
 }
 
 #ifndef NDEBUG
 static bool hasAnyPackExpansions(ArrayRef<TemplateArgument> Args) {
   for (const TemplateArgument &Arg : Args)
     if (Arg.isPackExpansion())
       return true;
 
   return true;
 }
 #endif
 
 QualType
 ASTContext::getTemplateSpecializationType(TemplateName Template,
                                           ArrayRef<TemplateArgument> Args,
                                           QualType Underlying) const {
   assert(!Template.getAsDependentTemplateName() &&
          "No dependent template names here!");
   // Look through qualified template names.
   if (QualifiedTemplateName *QTN = Template.getAsQualifiedTemplateName())
     Template = TemplateName(QTN->getTemplateDecl());
 
   bool IsTypeAlias =
     Template.getAsTemplateDecl() &&
     isa<TypeAliasTemplateDecl>(Template.getAsTemplateDecl());
   QualType CanonType;
   if (!Underlying.isNull())
     CanonType = getCanonicalType(Underlying);
   else {
     // We can get here with an alias template when the specialization contains
     // a pack expansion that does not match up with a parameter pack.
     assert((!IsTypeAlias || hasAnyPackExpansions(Args)) &&
            "Caller must compute aliased type");
     IsTypeAlias = false;
     CanonType = getCanonicalTemplateSpecializationType(Template, Args);
   }
 
   // Allocate the (non-canonical) template specialization type, but don't
   // try to unique it: these types typically have location information that
   // we don't unique and don't want to lose.
   void *Mem = Allocate(sizeof(TemplateSpecializationType) +
                        sizeof(TemplateArgument) * Args.size() +
                        (IsTypeAlias? sizeof(QualType) : 0),
                        TypeAlignment);
   auto *Spec
     = new (Mem) TemplateSpecializationType(Template, Args, CanonType,
                                          IsTypeAlias ? Underlying : QualType());
 
   Types.push_back(Spec);
   return QualType(Spec, 0);
 }
 
 static bool
 getCanonicalTemplateArguments(const ASTContext &C,
                               ArrayRef<TemplateArgument> OrigArgs,
                               SmallVectorImpl<TemplateArgument> &CanonArgs) {
   bool AnyNonCanonArgs = false;
   unsigned NumArgs = OrigArgs.size();
   CanonArgs.resize(NumArgs);
   for (unsigned I = 0; I != NumArgs; ++I) {
     const TemplateArgument &OrigArg = OrigArgs[I];
     TemplateArgument &CanonArg = CanonArgs[I];
     CanonArg = C.getCanonicalTemplateArgument(OrigArg);
     if (!CanonArg.structurallyEquals(OrigArg))
       AnyNonCanonArgs = true;
   }
   return AnyNonCanonArgs;
 }
 
 QualType ASTContext::getCanonicalTemplateSpecializationType(
     TemplateName Template, ArrayRef<TemplateArgument> Args) const {
   assert(!Template.getAsDependentTemplateName() &&
          "No dependent template names here!");
 
   // Look through qualified template names.
   if (QualifiedTemplateName *QTN = Template.getAsQualifiedTemplateName())
     Template = TemplateName(QTN->getTemplateDecl());
 
   // Build the canonical template specialization type.
   TemplateName CanonTemplate = getCanonicalTemplateName(Template);
   SmallVector<TemplateArgument, 4> CanonArgs;
   ::getCanonicalTemplateArguments(*this, Args, CanonArgs);
 
   // Determine whether this canonical template specialization type already
   // exists.
   llvm::FoldingSetNodeID ID;
   TemplateSpecializationType::Profile(ID, CanonTemplate,
                                       CanonArgs, *this);
 
   void *InsertPos = nullptr;
   TemplateSpecializationType *Spec
     = TemplateSpecializationTypes.FindNodeOrInsertPos(ID, InsertPos);
 
   if (!Spec) {
     // Allocate a new canonical template specialization type.
     void *Mem = Allocate((sizeof(TemplateSpecializationType) +
                           sizeof(TemplateArgument) * CanonArgs.size()),
                          TypeAlignment);
     Spec = new (Mem) TemplateSpecializationType(CanonTemplate,
                                                 CanonArgs,
                                                 QualType(), QualType());
     Types.push_back(Spec);
     TemplateSpecializationTypes.InsertNode(Spec, InsertPos);
   }
 
   assert(Spec->isDependentType() &&
          "Non-dependent template-id type must have a canonical type");
   return QualType(Spec, 0);
 }
 
 QualType ASTContext::getElaboratedType(ElaboratedTypeKeyword Keyword,
                                        NestedNameSpecifier *NNS,
                                        QualType NamedType,
                                        TagDecl *OwnedTagDecl) const {
   llvm::FoldingSetNodeID ID;
   ElaboratedType::Profile(ID, Keyword, NNS, NamedType, OwnedTagDecl);
 
   void *InsertPos = nullptr;
   ElaboratedType *T = ElaboratedTypes.FindNodeOrInsertPos(ID, InsertPos);
   if (T)
     return QualType(T, 0);
 
   QualType Canon = NamedType;
   if (!Canon.isCanonical()) {
     Canon = getCanonicalType(NamedType);
     ElaboratedType *CheckT = ElaboratedTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!CheckT && "Elaborated canonical type broken");
     (void)CheckT;
   }
 
   void *Mem = Allocate(ElaboratedType::totalSizeToAlloc<TagDecl *>(!!OwnedTagDecl),
                        TypeAlignment);
   T = new (Mem) ElaboratedType(Keyword, NNS, NamedType, Canon, OwnedTagDecl);
 
   Types.push_back(T);
   ElaboratedTypes.InsertNode(T, InsertPos);
   return QualType(T, 0);
 }
 
 QualType
 ASTContext::getParenType(QualType InnerType) const {
   llvm::FoldingSetNodeID ID;
   ParenType::Profile(ID, InnerType);
 
   void *InsertPos = nullptr;
   ParenType *T = ParenTypes.FindNodeOrInsertPos(ID, InsertPos);
   if (T)
     return QualType(T, 0);
 
   QualType Canon = InnerType;
   if (!Canon.isCanonical()) {
     Canon = getCanonicalType(InnerType);
     ParenType *CheckT = ParenTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!CheckT && "Paren canonical type broken");
     (void)CheckT;
   }
 
   T = new (*this, TypeAlignment) ParenType(InnerType, Canon);
   Types.push_back(T);
   ParenTypes.InsertNode(T, InsertPos);
   return QualType(T, 0);
 }
 
 QualType
 ASTContext::getMacroQualifiedType(QualType UnderlyingTy,
                                   const IdentifierInfo *MacroII) const {
   QualType Canon = UnderlyingTy;
   if (!Canon.isCanonical())
     Canon = getCanonicalType(UnderlyingTy);
 
   auto *newType = new (*this, TypeAlignment)
       MacroQualifiedType(UnderlyingTy, Canon, MacroII);
   Types.push_back(newType);
   return QualType(newType, 0);
 }
 
 QualType ASTContext::getDependentNameType(ElaboratedTypeKeyword Keyword,
                                           NestedNameSpecifier *NNS,
                                           const IdentifierInfo *Name,
                                           QualType Canon) const {
   if (Canon.isNull()) {
     NestedNameSpecifier *CanonNNS = getCanonicalNestedNameSpecifier(NNS);
     if (CanonNNS != NNS)
       Canon = getDependentNameType(Keyword, CanonNNS, Name);
   }
 
   llvm::FoldingSetNodeID ID;
   DependentNameType::Profile(ID, Keyword, NNS, Name);
 
   void *InsertPos = nullptr;
   DependentNameType *T
     = DependentNameTypes.FindNodeOrInsertPos(ID, InsertPos);
   if (T)
     return QualType(T, 0);
 
   T = new (*this, TypeAlignment) DependentNameType(Keyword, NNS, Name, Canon);
   Types.push_back(T);
   DependentNameTypes.InsertNode(T, InsertPos);
   return QualType(T, 0);
 }
 
 QualType
 ASTContext::getDependentTemplateSpecializationType(
                                  ElaboratedTypeKeyword Keyword,
                                  NestedNameSpecifier *NNS,
                                  const IdentifierInfo *Name,
                                  const TemplateArgumentListInfo &Args) const {
   // TODO: avoid this copy
   SmallVector<TemplateArgument, 16> ArgCopy;
   for (unsigned I = 0, E = Args.size(); I != E; ++I)
     ArgCopy.push_back(Args[I].getArgument());
   return getDependentTemplateSpecializationType(Keyword, NNS, Name, ArgCopy);
 }
 
 QualType
 ASTContext::getDependentTemplateSpecializationType(
                                  ElaboratedTypeKeyword Keyword,
                                  NestedNameSpecifier *NNS,
                                  const IdentifierInfo *Name,
                                  ArrayRef<TemplateArgument> Args) const {
   assert((!NNS || NNS->isDependent()) &&
          "nested-name-specifier must be dependent");
 
   llvm::FoldingSetNodeID ID;
   DependentTemplateSpecializationType::Profile(ID, *this, Keyword, NNS,
                                                Name, Args);
 
   void *InsertPos = nullptr;
   DependentTemplateSpecializationType *T
     = DependentTemplateSpecializationTypes.FindNodeOrInsertPos(ID, InsertPos);
   if (T)
     return QualType(T, 0);
 
   NestedNameSpecifier *CanonNNS = getCanonicalNestedNameSpecifier(NNS);
 
   ElaboratedTypeKeyword CanonKeyword = Keyword;
   if (Keyword == ETK_None) CanonKeyword = ETK_Typename;
 
   SmallVector<TemplateArgument, 16> CanonArgs;
   bool AnyNonCanonArgs =
       ::getCanonicalTemplateArguments(*this, Args, CanonArgs);
 
   QualType Canon;
   if (AnyNonCanonArgs || CanonNNS != NNS || CanonKeyword != Keyword) {
     Canon = getDependentTemplateSpecializationType(CanonKeyword, CanonNNS,
                                                    Name,
                                                    CanonArgs);
 
     // Find the insert position again.
     DependentTemplateSpecializationTypes.FindNodeOrInsertPos(ID, InsertPos);
   }
 
   void *Mem = Allocate((sizeof(DependentTemplateSpecializationType) +
                         sizeof(TemplateArgument) * Args.size()),
                        TypeAlignment);
   T = new (Mem) DependentTemplateSpecializationType(Keyword, NNS,
                                                     Name, Args, Canon);
   Types.push_back(T);
   DependentTemplateSpecializationTypes.InsertNode(T, InsertPos);
   return QualType(T, 0);
 }
 
 TemplateArgument ASTContext::getInjectedTemplateArg(NamedDecl *Param) {
   TemplateArgument Arg;
   if (const auto *TTP = dyn_cast<TemplateTypeParmDecl>(Param)) {
     QualType ArgType = getTypeDeclType(TTP);
     if (TTP->isParameterPack())
       ArgType = getPackExpansionType(ArgType, None);
 
     Arg = TemplateArgument(ArgType);
   } else if (auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(Param)) {
     QualType T =
         NTTP->getType().getNonPackExpansionType().getNonLValueExprType(*this);
     // For class NTTPs, ensure we include the 'const' so the type matches that
     // of a real template argument.
     // FIXME: It would be more faithful to model this as something like an
     // lvalue-to-rvalue conversion applied to a const-qualified lvalue.
     if (T->isRecordType())
       T.addConst();
     Expr *E = new (*this) DeclRefExpr(
         *this, NTTP, /*enclosing*/ false, T,
         Expr::getValueKindForType(NTTP->getType()), NTTP->getLocation());
 
     if (NTTP->isParameterPack())
       E = new (*this) PackExpansionExpr(DependentTy, E, NTTP->getLocation(),
                                         None);
     Arg = TemplateArgument(E);
   } else {
     auto *TTP = cast<TemplateTemplateParmDecl>(Param);
     if (TTP->isParameterPack())
       Arg = TemplateArgument(TemplateName(TTP), Optional<unsigned>());
     else
       Arg = TemplateArgument(TemplateName(TTP));
   }
 
   if (Param->isTemplateParameterPack())
     Arg = TemplateArgument::CreatePackCopy(*this, Arg);
 
   return Arg;
 }
 
 void
 ASTContext::getInjectedTemplateArgs(const TemplateParameterList *Params,
                                     SmallVectorImpl<TemplateArgument> &Args) {
   Args.reserve(Args.size() + Params->size());
 
   for (NamedDecl *Param : *Params)
     Args.push_back(getInjectedTemplateArg(Param));
 }
 
 QualType ASTContext::getPackExpansionType(QualType Pattern,
                                           Optional<unsigned> NumExpansions,
                                           bool ExpectPackInType) {
   assert((!ExpectPackInType || Pattern->containsUnexpandedParameterPack()) &&
          "Pack expansions must expand one or more parameter packs");
 
   llvm::FoldingSetNodeID ID;
   PackExpansionType::Profile(ID, Pattern, NumExpansions);
 
   void *InsertPos = nullptr;
   PackExpansionType *T = PackExpansionTypes.FindNodeOrInsertPos(ID, InsertPos);
   if (T)
     return QualType(T, 0);
 
   QualType Canon;
   if (!Pattern.isCanonical()) {
     Canon = getPackExpansionType(getCanonicalType(Pattern), NumExpansions,
                                  /*ExpectPackInType=*/false);
 
     // Find the insert position again, in case we inserted an element into
     // PackExpansionTypes and invalidated our insert position.
     PackExpansionTypes.FindNodeOrInsertPos(ID, InsertPos);
   }
 
   T = new (*this, TypeAlignment)
       PackExpansionType(Pattern, Canon, NumExpansions);
   Types.push_back(T);
   PackExpansionTypes.InsertNode(T, InsertPos);
   return QualType(T, 0);
 }
 
 /// CmpProtocolNames - Comparison predicate for sorting protocols
 /// alphabetically.
 static int CmpProtocolNames(ObjCProtocolDecl *const *LHS,
                             ObjCProtocolDecl *const *RHS) {
   return DeclarationName::compare((*LHS)->getDeclName(), (*RHS)->getDeclName());
 }
 
 static bool areSortedAndUniqued(ArrayRef<ObjCProtocolDecl *> Protocols) {
   if (Protocols.empty()) return true;
 
   if (Protocols[0]->getCanonicalDecl() != Protocols[0])
     return false;
 
   for (unsigned i = 1; i != Protocols.size(); ++i)
     if (CmpProtocolNames(&Protocols[i - 1], &Protocols[i]) >= 0 ||
         Protocols[i]->getCanonicalDecl() != Protocols[i])
       return false;
   return true;
 }
 
 static void
 SortAndUniqueProtocols(SmallVectorImpl<ObjCProtocolDecl *> &Protocols) {
   // Sort protocols, keyed by name.
   llvm::array_pod_sort(Protocols.begin(), Protocols.end(), CmpProtocolNames);
 
   // Canonicalize.
   for (ObjCProtocolDecl *&P : Protocols)
     P = P->getCanonicalDecl();
 
   // Remove duplicates.
   auto ProtocolsEnd = std::unique(Protocols.begin(), Protocols.end());
   Protocols.erase(ProtocolsEnd, Protocols.end());
 }
 
 QualType ASTContext::getObjCObjectType(QualType BaseType,
                                        ObjCProtocolDecl * const *Protocols,
                                        unsigned NumProtocols) const {
   return getObjCObjectType(BaseType, {},
                            llvm::makeArrayRef(Protocols, NumProtocols),
                            /*isKindOf=*/false);
 }
 
 QualType ASTContext::getObjCObjectType(
            QualType baseType,
            ArrayRef<QualType> typeArgs,
            ArrayRef<ObjCProtocolDecl *> protocols,
            bool isKindOf) const {
   // If the base type is an interface and there aren't any protocols or
   // type arguments to add, then the interface type will do just fine.
   if (typeArgs.empty() && protocols.empty() && !isKindOf &&
       isa<ObjCInterfaceType>(baseType))
     return baseType;
 
   // Look in the folding set for an existing type.
   llvm::FoldingSetNodeID ID;
   ObjCObjectTypeImpl::Profile(ID, baseType, typeArgs, protocols, isKindOf);
   void *InsertPos = nullptr;
   if (ObjCObjectType *QT = ObjCObjectTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(QT, 0);
 
   // Determine the type arguments to be used for canonicalization,
   // which may be explicitly specified here or written on the base
   // type.
   ArrayRef<QualType> effectiveTypeArgs = typeArgs;
   if (effectiveTypeArgs.empty()) {
     if (const auto *baseObject = baseType->getAs<ObjCObjectType>())
       effectiveTypeArgs = baseObject->getTypeArgs();
   }
 
   // Build the canonical type, which has the canonical base type and a
   // sorted-and-uniqued list of protocols and the type arguments
   // canonicalized.
   QualType canonical;
   bool typeArgsAreCanonical = llvm::all_of(
       effectiveTypeArgs, [&](QualType type) { return type.isCanonical(); });
   bool protocolsSorted = areSortedAndUniqued(protocols);
   if (!typeArgsAreCanonical || !protocolsSorted || !baseType.isCanonical()) {
     // Determine the canonical type arguments.
     ArrayRef<QualType> canonTypeArgs;
     SmallVector<QualType, 4> canonTypeArgsVec;
     if (!typeArgsAreCanonical) {
       canonTypeArgsVec.reserve(effectiveTypeArgs.size());
       for (auto typeArg : effectiveTypeArgs)
         canonTypeArgsVec.push_back(getCanonicalType(typeArg));
       canonTypeArgs = canonTypeArgsVec;
     } else {
       canonTypeArgs = effectiveTypeArgs;
     }
 
     ArrayRef<ObjCProtocolDecl *> canonProtocols;
     SmallVector<ObjCProtocolDecl*, 8> canonProtocolsVec;
     if (!protocolsSorted) {
       canonProtocolsVec.append(protocols.begin(), protocols.end());
       SortAndUniqueProtocols(canonProtocolsVec);
       canonProtocols = canonProtocolsVec;
     } else {
       canonProtocols = protocols;
     }
 
     canonical = getObjCObjectType(getCanonicalType(baseType), canonTypeArgs,
                                   canonProtocols, isKindOf);
 
     // Regenerate InsertPos.
     ObjCObjectTypes.FindNodeOrInsertPos(ID, InsertPos);
   }
 
   unsigned size = sizeof(ObjCObjectTypeImpl);
   size += typeArgs.size() * sizeof(QualType);
   size += protocols.size() * sizeof(ObjCProtocolDecl *);
   void *mem = Allocate(size, TypeAlignment);
   auto *T =
     new (mem) ObjCObjectTypeImpl(canonical, baseType, typeArgs, protocols,
                                  isKindOf);
 
   Types.push_back(T);
   ObjCObjectTypes.InsertNode(T, InsertPos);
   return QualType(T, 0);
 }
 
 /// Apply Objective-C protocol qualifiers to the given type.
 /// If this is for the canonical type of a type parameter, we can apply
 /// protocol qualifiers on the ObjCObjectPointerType.
 QualType
 ASTContext::applyObjCProtocolQualifiers(QualType type,
                   ArrayRef<ObjCProtocolDecl *> protocols, bool &hasError,
                   bool allowOnPointerType) const {
   hasError = false;
 
   if (const auto *objT = dyn_cast<ObjCTypeParamType>(type.getTypePtr())) {
     return getObjCTypeParamType(objT->getDecl(), protocols);
   }
 
   // Apply protocol qualifiers to ObjCObjectPointerType.
   if (allowOnPointerType) {
     if (const auto *objPtr =
             dyn_cast<ObjCObjectPointerType>(type.getTypePtr())) {
       const ObjCObjectType *objT = objPtr->getObjectType();
       // Merge protocol lists and construct ObjCObjectType.
       SmallVector<ObjCProtocolDecl*, 8> protocolsVec;
       protocolsVec.append(objT->qual_begin(),
                           objT->qual_end());
       protocolsVec.append(protocols.begin(), protocols.end());
       ArrayRef<ObjCProtocolDecl *> protocols = protocolsVec;
       type = getObjCObjectType(
              objT->getBaseType(),
              objT->getTypeArgsAsWritten(),
              protocols,
              objT->isKindOfTypeAsWritten());
       return getObjCObjectPointerType(type);
     }
   }
 
   // Apply protocol qualifiers to ObjCObjectType.
   if (const auto *objT = dyn_cast<ObjCObjectType>(type.getTypePtr())){
     // FIXME: Check for protocols to which the class type is already
     // known to conform.
 
     return getObjCObjectType(objT->getBaseType(),
                              objT->getTypeArgsAsWritten(),
                              protocols,
                              objT->isKindOfTypeAsWritten());
   }
 
   // If the canonical type is ObjCObjectType, ...
   if (type->isObjCObjectType()) {
     // Silently overwrite any existing protocol qualifiers.
     // TODO: determine whether that's the right thing to do.
 
     // FIXME: Check for protocols to which the class type is already
     // known to conform.
     return getObjCObjectType(type, {}, protocols, false);
   }
 
   // id<protocol-list>
   if (type->isObjCIdType()) {
     const auto *objPtr = type->castAs<ObjCObjectPointerType>();
     type = getObjCObjectType(ObjCBuiltinIdTy, {}, protocols,
                                  objPtr->isKindOfType());
     return getObjCObjectPointerType(type);
   }
 
   // Class<protocol-list>
   if (type->isObjCClassType()) {
     const auto *objPtr = type->castAs<ObjCObjectPointerType>();
     type = getObjCObjectType(ObjCBuiltinClassTy, {}, protocols,
                                  objPtr->isKindOfType());
     return getObjCObjectPointerType(type);
   }
 
   hasError = true;
   return type;
 }
 
 QualType
 ASTContext::getObjCTypeParamType(const ObjCTypeParamDecl *Decl,
                                  ArrayRef<ObjCProtocolDecl *> protocols) const {
   // Look in the folding set for an existing type.
   llvm::FoldingSetNodeID ID;
   ObjCTypeParamType::Profile(ID, Decl, Decl->getUnderlyingType(), protocols);
   void *InsertPos = nullptr;
   if (ObjCTypeParamType *TypeParam =
       ObjCTypeParamTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(TypeParam, 0);
 
   // We canonicalize to the underlying type.
   QualType Canonical = getCanonicalType(Decl->getUnderlyingType());
   if (!protocols.empty()) {
     // Apply the protocol qualifers.
     bool hasError;
     Canonical = getCanonicalType(applyObjCProtocolQualifiers(
         Canonical, protocols, hasError, true /*allowOnPointerType*/));
     assert(!hasError && "Error when apply protocol qualifier to bound type");
   }
 
   unsigned size = sizeof(ObjCTypeParamType);
   size += protocols.size() * sizeof(ObjCProtocolDecl *);
   void *mem = Allocate(size, TypeAlignment);
   auto *newType = new (mem) ObjCTypeParamType(Decl, Canonical, protocols);
 
   Types.push_back(newType);
   ObjCTypeParamTypes.InsertNode(newType, InsertPos);
   return QualType(newType, 0);
 }
 
 void ASTContext::adjustObjCTypeParamBoundType(const ObjCTypeParamDecl *Orig,
                                               ObjCTypeParamDecl *New) const {
   New->setTypeSourceInfo(getTrivialTypeSourceInfo(Orig->getUnderlyingType()));
   // Update TypeForDecl after updating TypeSourceInfo.
   auto NewTypeParamTy = cast<ObjCTypeParamType>(New->getTypeForDecl());
   SmallVector<ObjCProtocolDecl *, 8> protocols;
   protocols.append(NewTypeParamTy->qual_begin(), NewTypeParamTy->qual_end());
   QualType UpdatedTy = getObjCTypeParamType(New, protocols);
   New->setTypeForDecl(UpdatedTy.getTypePtr());
 }
 
 /// ObjCObjectAdoptsQTypeProtocols - Checks that protocols in IC's
 /// protocol list adopt all protocols in QT's qualified-id protocol
 /// list.
 bool ASTContext::ObjCObjectAdoptsQTypeProtocols(QualType QT,
                                                 ObjCInterfaceDecl *IC) {
   if (!QT->isObjCQualifiedIdType())
     return false;
 
   if (const auto *OPT = QT->getAs<ObjCObjectPointerType>()) {
     // If both the right and left sides have qualifiers.
     for (auto *Proto : OPT->quals()) {
       if (!IC->ClassImplementsProtocol(Proto, false))
         return false;
     }
     return true;
   }
   return false;
 }
 
 /// QIdProtocolsAdoptObjCObjectProtocols - Checks that protocols in
 /// QT's qualified-id protocol list adopt all protocols in IDecl's list
 /// of protocols.
 bool ASTContext::QIdProtocolsAdoptObjCObjectProtocols(QualType QT,
                                                 ObjCInterfaceDecl *IDecl) {
   if (!QT->isObjCQualifiedIdType())
     return false;
   const auto *OPT = QT->getAs<ObjCObjectPointerType>();
   if (!OPT)
     return false;
   if (!IDecl->hasDefinition())
     return false;
   llvm::SmallPtrSet<ObjCProtocolDecl *, 8> InheritedProtocols;
   CollectInheritedProtocols(IDecl, InheritedProtocols);
   if (InheritedProtocols.empty())
     return false;
   // Check that if every protocol in list of id<plist> conforms to a protocol
   // of IDecl's, then bridge casting is ok.
   bool Conforms = false;
   for (auto *Proto : OPT->quals()) {
     Conforms = false;
     for (auto *PI : InheritedProtocols) {
       if (ProtocolCompatibleWithProtocol(Proto, PI)) {
         Conforms = true;
         break;
       }
     }
     if (!Conforms)
       break;
   }
   if (Conforms)
     return true;
 
   for (auto *PI : InheritedProtocols) {
     // If both the right and left sides have qualifiers.
     bool Adopts = false;
     for (auto *Proto : OPT->quals()) {
       // return 'true' if 'PI' is in the inheritance hierarchy of Proto
       if ((Adopts = ProtocolCompatibleWithProtocol(PI, Proto)))
         break;
     }
     if (!Adopts)
       return false;
   }
   return true;
 }
 
 /// getObjCObjectPointerType - Return a ObjCObjectPointerType type for
 /// the given object type.
 QualType ASTContext::getObjCObjectPointerType(QualType ObjectT) const {
   llvm::FoldingSetNodeID ID;
   ObjCObjectPointerType::Profile(ID, ObjectT);
 
   void *InsertPos = nullptr;
   if (ObjCObjectPointerType *QT =
               ObjCObjectPointerTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(QT, 0);
 
   // Find the canonical object type.
   QualType Canonical;
   if (!ObjectT.isCanonical()) {
     Canonical = getObjCObjectPointerType(getCanonicalType(ObjectT));
 
     // Regenerate InsertPos.
     ObjCObjectPointerTypes.FindNodeOrInsertPos(ID, InsertPos);
   }
 
   // No match.
   void *Mem = Allocate(sizeof(ObjCObjectPointerType), TypeAlignment);
   auto *QType =
     new (Mem) ObjCObjectPointerType(Canonical, ObjectT);
 
   Types.push_back(QType);
   ObjCObjectPointerTypes.InsertNode(QType, InsertPos);
   return QualType(QType, 0);
 }
 
 /// getObjCInterfaceType - Return the unique reference to the type for the
 /// specified ObjC interface decl. The list of protocols is optional.
 QualType ASTContext::getObjCInterfaceType(const ObjCInterfaceDecl *Decl,
                                           ObjCInterfaceDecl *PrevDecl) const {
   if (Decl->TypeForDecl)
     return QualType(Decl->TypeForDecl, 0);
 
   if (PrevDecl) {
     assert(PrevDecl->TypeForDecl && "previous decl has no TypeForDecl");
     Decl->TypeForDecl = PrevDecl->TypeForDecl;
     return QualType(PrevDecl->TypeForDecl, 0);
   }
 
   // Prefer the definition, if there is one.
   if (const ObjCInterfaceDecl *Def = Decl->getDefinition())
     Decl = Def;
 
   void *Mem = Allocate(sizeof(ObjCInterfaceType), TypeAlignment);
   auto *T = new (Mem) ObjCInterfaceType(Decl);
   Decl->TypeForDecl = T;
   Types.push_back(T);
   return QualType(T, 0);
 }
 
 /// getTypeOfExprType - Unlike many "get<Type>" functions, we can't unique
 /// TypeOfExprType AST's (since expression's are never shared). For example,
 /// multiple declarations that refer to "typeof(x)" all contain different
 /// DeclRefExpr's. This doesn't effect the type checker, since it operates
 /// on canonical type's (which are always unique).
 QualType ASTContext::getTypeOfExprType(Expr *tofExpr) const {
   TypeOfExprType *toe;
   if (tofExpr->isTypeDependent()) {
     llvm::FoldingSetNodeID ID;
     DependentTypeOfExprType::Profile(ID, *this, tofExpr);
 
     void *InsertPos = nullptr;
     DependentTypeOfExprType *Canon
       = DependentTypeOfExprTypes.FindNodeOrInsertPos(ID, InsertPos);
     if (Canon) {
       // We already have a "canonical" version of an identical, dependent
       // typeof(expr) type. Use that as our canonical type.
       toe = new (*this, TypeAlignment) TypeOfExprType(tofExpr,
                                           QualType((TypeOfExprType*)Canon, 0));
     } else {
       // Build a new, canonical typeof(expr) type.
       Canon
         = new (*this, TypeAlignment) DependentTypeOfExprType(*this, tofExpr);
       DependentTypeOfExprTypes.InsertNode(Canon, InsertPos);
       toe = Canon;
     }
   } else {
     QualType Canonical = getCanonicalType(tofExpr->getType());
     toe = new (*this, TypeAlignment) TypeOfExprType(tofExpr, Canonical);
   }
   Types.push_back(toe);
   return QualType(toe, 0);
 }
 
 /// getTypeOfType -  Unlike many "get<Type>" functions, we don't unique
 /// TypeOfType nodes. The only motivation to unique these nodes would be
 /// memory savings. Since typeof(t) is fairly uncommon, space shouldn't be
 /// an issue. This doesn't affect the type checker, since it operates
 /// on canonical types (which are always unique).
 QualType ASTContext::getTypeOfType(QualType tofType) const {
   QualType Canonical = getCanonicalType(tofType);
   auto *tot = new (*this, TypeAlignment) TypeOfType(tofType, Canonical);
   Types.push_back(tot);
   return QualType(tot, 0);
 }
 
 /// getReferenceQualifiedType - Given an expr, will return the type for
 /// that expression, as in [dcl.type.simple]p4 but without taking id-expressions
 /// and class member access into account.
 QualType ASTContext::getReferenceQualifiedType(const Expr *E) const {
   // C++11 [dcl.type.simple]p4:
   //   [...]
   QualType T = E->getType();
   switch (E->getValueKind()) {
   //     - otherwise, if e is an xvalue, decltype(e) is T&&, where T is the
   //       type of e;
   case VK_XValue:
     return getRValueReferenceType(T);
   //     - otherwise, if e is an lvalue, decltype(e) is T&, where T is the
   //       type of e;
   case VK_LValue:
     return getLValueReferenceType(T);
   //  - otherwise, decltype(e) is the type of e.
   case VK_PRValue:
     return T;
   }
   llvm_unreachable("Unknown value kind");
 }
 
 /// Unlike many "get<Type>" functions, we don't unique DecltypeType
 /// nodes. This would never be helpful, since each such type has its own
 /// expression, and would not give a significant memory saving, since there
 /// is an Expr tree under each such type.
 QualType ASTContext::getDecltypeType(Expr *e, QualType UnderlyingType) const {
   DecltypeType *dt;
 
   // C++11 [temp.type]p2:
   //   If an expression e involves a template parameter, decltype(e) denotes a
   //   unique dependent type. Two such decltype-specifiers refer to the same
   //   type only if their expressions are equivalent (14.5.6.1).
   if (e->isInstantiationDependent()) {
     llvm::FoldingSetNodeID ID;
     DependentDecltypeType::Profile(ID, *this, e);
 
     void *InsertPos = nullptr;
     DependentDecltypeType *Canon
       = DependentDecltypeTypes.FindNodeOrInsertPos(ID, InsertPos);
     if (!Canon) {
       // Build a new, canonical decltype(expr) type.
       Canon = new (*this, TypeAlignment) DependentDecltypeType(*this, e);
       DependentDecltypeTypes.InsertNode(Canon, InsertPos);
     }
     dt = new (*this, TypeAlignment)
         DecltypeType(e, UnderlyingType, QualType((DecltypeType *)Canon, 0));
   } else {
     dt = new (*this, TypeAlignment)
         DecltypeType(e, UnderlyingType, getCanonicalType(UnderlyingType));
   }
   Types.push_back(dt);
   return QualType(dt, 0);
 }
 
 /// getUnaryTransformationType - We don't unique these, since the memory
 /// savings are minimal and these are rare.
 QualType ASTContext::getUnaryTransformType(QualType BaseType,
                                            QualType UnderlyingType,
                                            UnaryTransformType::UTTKind Kind)
     const {
   UnaryTransformType *ut = nullptr;
 
   if (BaseType->isDependentType()) {
     // Look in the folding set for an existing type.
     llvm::FoldingSetNodeID ID;
     DependentUnaryTransformType::Profile(ID, getCanonicalType(BaseType), Kind);
 
     void *InsertPos = nullptr;
     DependentUnaryTransformType *Canon
       = DependentUnaryTransformTypes.FindNodeOrInsertPos(ID, InsertPos);
 
     if (!Canon) {
       // Build a new, canonical __underlying_type(type) type.
       Canon = new (*this, TypeAlignment)
              DependentUnaryTransformType(*this, getCanonicalType(BaseType),
                                          Kind);
       DependentUnaryTransformTypes.InsertNode(Canon, InsertPos);
     }
     ut = new (*this, TypeAlignment) UnaryTransformType (BaseType,
                                                         QualType(), Kind,
                                                         QualType(Canon, 0));
   } else {
     QualType CanonType = getCanonicalType(UnderlyingType);
     ut = new (*this, TypeAlignment) UnaryTransformType (BaseType,
                                                         UnderlyingType, Kind,
                                                         CanonType);
   }
   Types.push_back(ut);
   return QualType(ut, 0);
 }
 
 QualType ASTContext::getAutoTypeInternal(
     QualType DeducedType, AutoTypeKeyword Keyword, bool IsDependent,
     bool IsPack, ConceptDecl *TypeConstraintConcept,
     ArrayRef<TemplateArgument> TypeConstraintArgs, bool IsCanon) const {
   if (DeducedType.isNull() && Keyword == AutoTypeKeyword::Auto &&
       !TypeConstraintConcept && !IsDependent)
     return getAutoDeductType();
 
   // Look in the folding set for an existing type.
   void *InsertPos = nullptr;
   llvm::FoldingSetNodeID ID;
   AutoType::Profile(ID, *this, DeducedType, Keyword, IsDependent,
                     TypeConstraintConcept, TypeConstraintArgs);
   if (AutoType *AT = AutoTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(AT, 0);
 
   QualType Canon;
   if (!IsCanon) {
     if (DeducedType.isNull()) {
       SmallVector<TemplateArgument, 4> CanonArgs;
       bool AnyNonCanonArgs =
           ::getCanonicalTemplateArguments(*this, TypeConstraintArgs, CanonArgs);
       if (AnyNonCanonArgs) {
         Canon = getAutoTypeInternal(QualType(), Keyword, IsDependent, IsPack,
                                     TypeConstraintConcept, CanonArgs, true);
         // Find the insert position again.
         AutoTypes.FindNodeOrInsertPos(ID, InsertPos);
       }
     } else {
       Canon = DeducedType.getCanonicalType();
     }
   }
 
   void *Mem = Allocate(sizeof(AutoType) +
                            sizeof(TemplateArgument) * TypeConstraintArgs.size(),
                        TypeAlignment);
   auto *AT = new (Mem) AutoType(
       DeducedType, Keyword,
       (IsDependent ? TypeDependence::DependentInstantiation
                    : TypeDependence::None) |
           (IsPack ? TypeDependence::UnexpandedPack : TypeDependence::None),
       Canon, TypeConstraintConcept, TypeConstraintArgs);
   Types.push_back(AT);
   AutoTypes.InsertNode(AT, InsertPos);
   return QualType(AT, 0);
 }
 
 /// getAutoType - Return the uniqued reference to the 'auto' type which has been
 /// deduced to the given type, or to the canonical undeduced 'auto' type, or the
 /// canonical deduced-but-dependent 'auto' type.
 QualType
 ASTContext::getAutoType(QualType DeducedType, AutoTypeKeyword Keyword,
                         bool IsDependent, bool IsPack,
                         ConceptDecl *TypeConstraintConcept,
                         ArrayRef<TemplateArgument> TypeConstraintArgs) const {
   assert((!IsPack || IsDependent) && "only use IsPack for a dependent pack");
   assert((!IsDependent || DeducedType.isNull()) &&
          "A dependent auto should be undeduced");
   return getAutoTypeInternal(DeducedType, Keyword, IsDependent, IsPack,
                              TypeConstraintConcept, TypeConstraintArgs);
 }
 
 /// Return the uniqued reference to the deduced template specialization type
 /// which has been deduced to the given type, or to the canonical undeduced
 /// such type, or the canonical deduced-but-dependent such type.
 QualType ASTContext::getDeducedTemplateSpecializationType(
     TemplateName Template, QualType DeducedType, bool IsDependent) const {
   // Look in the folding set for an existing type.
   void *InsertPos = nullptr;
   llvm::FoldingSetNodeID ID;
   DeducedTemplateSpecializationType::Profile(ID, Template, DeducedType,
                                              IsDependent);
   if (DeducedTemplateSpecializationType *DTST =
           DeducedTemplateSpecializationTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(DTST, 0);
 
   auto *DTST = new (*this, TypeAlignment)
       DeducedTemplateSpecializationType(Template, DeducedType, IsDependent);
   llvm::FoldingSetNodeID TempID;
   DTST->Profile(TempID);
   assert(ID == TempID && "ID does not match");
   Types.push_back(DTST);
   DeducedTemplateSpecializationTypes.InsertNode(DTST, InsertPos);
   return QualType(DTST, 0);
 }
 
 /// getAtomicType - Return the uniqued reference to the atomic type for
 /// the given value type.
 QualType ASTContext::getAtomicType(QualType T) const {
   // Unique pointers, to guarantee there is only one pointer of a particular
   // structure.
   llvm::FoldingSetNodeID ID;
   AtomicType::Profile(ID, T);
 
   void *InsertPos = nullptr;
   if (AtomicType *AT = AtomicTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(AT, 0);
 
   // If the atomic value type isn't canonical, this won't be a canonical type
   // either, so fill in the canonical type field.
   QualType Canonical;
   if (!T.isCanonical()) {
     Canonical = getAtomicType(getCanonicalType(T));
 
     // Get the new insert position for the node we care about.
     AtomicType *NewIP = AtomicTypes.FindNodeOrInsertPos(ID, InsertPos);
     assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP;
   }
   auto *New = new (*this, TypeAlignment) AtomicType(T, Canonical);
   Types.push_back(New);
   AtomicTypes.InsertNode(New, InsertPos);
   return QualType(New, 0);
 }
 
 /// getAutoDeductType - Get type pattern for deducing against 'auto'.
 QualType ASTContext::getAutoDeductType() const {
   if (AutoDeductTy.isNull())
     AutoDeductTy = QualType(new (*this, TypeAlignment)
                                 AutoType(QualType(), AutoTypeKeyword::Auto,
                                          TypeDependence::None, QualType(),
                                          /*concept*/ nullptr, /*args*/ {}),
                             0);
   return AutoDeductTy;
 }
 
 /// getAutoRRefDeductType - Get type pattern for deducing against 'auto &&'.
 QualType ASTContext::getAutoRRefDeductType() const {
   if (AutoRRefDeductTy.isNull())
     AutoRRefDeductTy = getRValueReferenceType(getAutoDeductType());
   assert(!AutoRRefDeductTy.isNull() && "can't build 'auto &&' pattern");
   return AutoRRefDeductTy;
 }
 
 /// getTagDeclType - Return the unique reference to the type for the
 /// specified TagDecl (struct/union/class/enum) decl.
 QualType ASTContext::getTagDeclType(const TagDecl *Decl) const {
   assert(Decl);
   // FIXME: What is the design on getTagDeclType when it requires casting
   // away const?  mutable?
   return getTypeDeclType(const_cast<TagDecl*>(Decl));
 }
 
 /// getSizeType - Return the unique type for "size_t" (C99 7.17), the result
 /// of the sizeof operator (C99 6.5.3.4p4). The value is target dependent and
 /// needs to agree with the definition in <stddef.h>.
 CanQualType ASTContext::getSizeType() const {
   return getFromTargetType(Target->getSizeType());
 }
 
 /// Return the unique signed counterpart of the integer type
 /// corresponding to size_t.
 CanQualType ASTContext::getSignedSizeType() const {
   return getFromTargetType(Target->getSignedSizeType());
 }
 
 /// getIntMaxType - Return the unique type for "intmax_t" (C99 7.18.1.5).
 CanQualType ASTContext::getIntMaxType() const {
   return getFromTargetType(Target->getIntMaxType());
 }
 
 /// getUIntMaxType - Return the unique type for "uintmax_t" (C99 7.18.1.5).
 CanQualType ASTContext::getUIntMaxType() const {
   return getFromTargetType(Target->getUIntMaxType());
 }
 
 /// getSignedWCharType - Return the type of "signed wchar_t".
 /// Used when in C++, as a GCC extension.
 QualType ASTContext::getSignedWCharType() const {
   // FIXME: derive from "Target" ?
   return WCharTy;
 }
 
 /// getUnsignedWCharType - Return the type of "unsigned wchar_t".
 /// Used when in C++, as a GCC extension.
 QualType ASTContext::getUnsignedWCharType() const {
   // FIXME: derive from "Target" ?
   return UnsignedIntTy;
 }
 
 QualType ASTContext::getIntPtrType() const {
   return getFromTargetType(Target->getIntPtrType());
 }
 
 QualType ASTContext::getUIntPtrType() const {
   return getCorrespondingUnsignedType(getIntPtrType());
 }
 
 /// getPointerDiffType - Return the unique type for "ptrdiff_t" (C99 7.17)
 /// defined in <stddef.h>. Pointer - pointer requires this (C99 6.5.6p9).
 QualType ASTContext::getPointerDiffType() const {
   return getFromTargetType(Target->getPtrDiffType(0));
 }
 
 /// Return the unique unsigned counterpart of "ptrdiff_t"
 /// integer type. The standard (C11 7.21.6.1p7) refers to this type
 /// in the definition of %tu format specifier.
 QualType ASTContext::getUnsignedPointerDiffType() const {
   return getFromTargetType(Target->getUnsignedPtrDiffType(0));
 }
 
 /// Return the unique type for "pid_t" defined in
 /// <sys/types.h>. We need this to compute the correct type for vfork().
 QualType ASTContext::getProcessIDType() const {
   return getFromTargetType(Target->getProcessIDType());
 }
 
 //===----------------------------------------------------------------------===//
 //                              Type Operators
 //===----------------------------------------------------------------------===//
 
 CanQualType ASTContext::getCanonicalParamType(QualType T) const {
   // Push qualifiers into arrays, and then discard any remaining
   // qualifiers.
   T = getCanonicalType(T);
   T = getVariableArrayDecayedType(T);
   const Type *Ty = T.getTypePtr();
   QualType Result;
   if (isa<ArrayType>(Ty)) {
     Result = getArrayDecayedType(QualType(Ty,0));
   } else if (isa<FunctionType>(Ty)) {
     Result = getPointerType(QualType(Ty, 0));
   } else {
     Result = QualType(Ty, 0);
   }
 
   return CanQualType::CreateUnsafe(Result);
 }
 
 QualType ASTContext::getUnqualifiedArrayType(QualType type,
                                              Qualifiers &quals) {
   SplitQualType splitType = type.getSplitUnqualifiedType();
 
   // FIXME: getSplitUnqualifiedType() actually walks all the way to
   // the unqualified desugared type and then drops it on the floor.
   // We then have to strip that sugar back off with
   // getUnqualifiedDesugaredType(), which is silly.
   const auto *AT =
       dyn_cast<ArrayType>(splitType.Ty->getUnqualifiedDesugaredType());
 
   // If we don't have an array, just use the results in splitType.
   if (!AT) {
     quals = splitType.Quals;
     return QualType(splitType.Ty, 0);
   }
 
   // Otherwise, recurse on the array's element type.
   QualType elementType = AT->getElementType();
   QualType unqualElementType = getUnqualifiedArrayType(elementType, quals);
 
   // If that didn't change the element type, AT has no qualifiers, so we
   // can just use the results in splitType.
   if (elementType == unqualElementType) {
     assert(quals.empty()); // from the recursive call
     quals = splitType.Quals;
     return QualType(splitType.Ty, 0);
   }
 
   // Otherwise, add in the qualifiers from the outermost type, then
   // build the type back up.
   quals.addConsistentQualifiers(splitType.Quals);
 
   if (const auto *CAT = dyn_cast<ConstantArrayType>(AT)) {
     return getConstantArrayType(unqualElementType, CAT->getSize(),
                                 CAT->getSizeExpr(), CAT->getSizeModifier(), 0);
   }
 
   if (const auto *IAT = dyn_cast<IncompleteArrayType>(AT)) {
     return getIncompleteArrayType(unqualElementType, IAT->getSizeModifier(), 0);
   }
 
   if (const auto *VAT = dyn_cast<VariableArrayType>(AT)) {
     return getVariableArrayType(unqualElementType,
                                 VAT->getSizeExpr(),
                                 VAT->getSizeModifier(),
                                 VAT->getIndexTypeCVRQualifiers(),
                                 VAT->getBracketsRange());
   }
 
   const auto *DSAT = cast<DependentSizedArrayType>(AT);
   return getDependentSizedArrayType(unqualElementType, DSAT->getSizeExpr(),
                                     DSAT->getSizeModifier(), 0,
                                     SourceRange());
 }
 
 /// Attempt to unwrap two types that may both be array types with the same bound
 /// (or both be array types of unknown bound) for the purpose of comparing the
 /// cv-decomposition of two types per C++ [conv.qual].
 ///
 /// \param AllowPiMismatch Allow the Pi1 and Pi2 to differ as described in
 ///        C++20 [conv.qual], if permitted by the current language mode.
 void ASTContext::UnwrapSimilarArrayTypes(QualType &T1, QualType &T2,
                                          bool AllowPiMismatch) {
   while (true) {
     auto *AT1 = getAsArrayType(T1);
     if (!AT1)
       return;
 
     auto *AT2 = getAsArrayType(T2);
     if (!AT2)
       return;
 
     // If we don't have two array types with the same constant bound nor two
     // incomplete array types, we've unwrapped everything we can.
     // C++20 also permits one type to be a constant array type and the other
     // to be an incomplete array type.
     // FIXME: Consider also unwrapping array of unknown bound and VLA.
     if (auto *CAT1 = dyn_cast<ConstantArrayType>(AT1)) {
       auto *CAT2 = dyn_cast<ConstantArrayType>(AT2);
       if (!((CAT2 && CAT1->getSize() == CAT2->getSize()) ||
             (AllowPiMismatch && getLangOpts().CPlusPlus20 &&
              isa<IncompleteArrayType>(AT2))))
         return;
     } else if (isa<IncompleteArrayType>(AT1)) {
       if (!(isa<IncompleteArrayType>(AT2) ||
             (AllowPiMismatch && getLangOpts().CPlusPlus20 &&
              isa<ConstantArrayType>(AT2))))
         return;
     } else {
       return;
     }
 
     T1 = AT1->getElementType();
     T2 = AT2->getElementType();
   }
 }
 
 /// Attempt to unwrap two types that may be similar (C++ [conv.qual]).
 ///
 /// If T1 and T2 are both pointer types of the same kind, or both array types
 /// with the same bound, unwraps layers from T1 and T2 until a pointer type is
 /// unwrapped. Top-level qualifiers on T1 and T2 are ignored.
 ///
 /// This function will typically be called in a loop that successively
 /// "unwraps" pointer and pointer-to-member types to compare them at each
 /// level.
 ///
 /// \param AllowPiMismatch Allow the Pi1 and Pi2 to differ as described in
 ///        C++20 [conv.qual], if permitted by the current language mode.
 ///
 /// \return \c true if a pointer type was unwrapped, \c false if we reached a
 /// pair of types that can't be unwrapped further.
 bool ASTContext::UnwrapSimilarTypes(QualType &T1, QualType &T2,
                                     bool AllowPiMismatch) {
   UnwrapSimilarArrayTypes(T1, T2, AllowPiMismatch);
 
   const auto *T1PtrType = T1->getAs<PointerType>();
   const auto *T2PtrType = T2->getAs<PointerType>();
   if (T1PtrType && T2PtrType) {
     T1 = T1PtrType->getPointeeType();
     T2 = T2PtrType->getPointeeType();
     return true;
   }
 
   const auto *T1MPType = T1->getAs<MemberPointerType>();
   const auto *T2MPType = T2->getAs<MemberPointerType>();
   if (T1MPType && T2MPType &&
       hasSameUnqualifiedType(QualType(T1MPType->getClass(), 0),
                              QualType(T2MPType->getClass(), 0))) {
     T1 = T1MPType->getPointeeType();
     T2 = T2MPType->getPointeeType();
     return true;
   }
 
   if (getLangOpts().ObjC) {
     const auto *T1OPType = T1->getAs<ObjCObjectPointerType>();
     const auto *T2OPType = T2->getAs<ObjCObjectPointerType>();
     if (T1OPType && T2OPType) {
       T1 = T1OPType->getPointeeType();
       T2 = T2OPType->getPointeeType();
       return true;
     }
   }
 
   // FIXME: Block pointers, too?
 
   return false;
 }
 
 bool ASTContext::hasSimilarType(QualType T1, QualType T2) {
   while (true) {
     Qualifiers Quals;
     T1 = getUnqualifiedArrayType(T1, Quals);
     T2 = getUnqualifiedArrayType(T2, Quals);
     if (hasSameType(T1, T2))
       return true;
     if (!UnwrapSimilarTypes(T1, T2))
       return false;
   }
 }
 
 bool ASTContext::hasCvrSimilarType(QualType T1, QualType T2) {
   while (true) {
     Qualifiers Quals1, Quals2;
     T1 = getUnqualifiedArrayType(T1, Quals1);
     T2 = getUnqualifiedArrayType(T2, Quals2);
 
     Quals1.removeCVRQualifiers();
     Quals2.removeCVRQualifiers();
     if (Quals1 != Quals2)
       return false;
 
     if (hasSameType(T1, T2))
       return true;
 
     if (!UnwrapSimilarTypes(T1, T2, /*AllowPiMismatch*/ false))
       return false;
   }
 }
 
 DeclarationNameInfo
 ASTContext::getNameForTemplate(TemplateName Name,
                                SourceLocation NameLoc) const {
   switch (Name.getKind()) {
   case TemplateName::QualifiedTemplate:
   case TemplateName::Template:
     // DNInfo work in progress: CHECKME: what about DNLoc?
     return DeclarationNameInfo(Name.getAsTemplateDecl()->getDeclName(),
                                NameLoc);
 
   case TemplateName::OverloadedTemplate: {
     OverloadedTemplateStorage *Storage = Name.getAsOverloadedTemplate();
     // DNInfo work in progress: CHECKME: what about DNLoc?
     return DeclarationNameInfo((*Storage->begin())->getDeclName(), NameLoc);
   }
 
   case TemplateName::AssumedTemplate: {
     AssumedTemplateStorage *Storage = Name.getAsAssumedTemplateName();
     return DeclarationNameInfo(Storage->getDeclName(), NameLoc);
   }
 
   case TemplateName::DependentTemplate: {
     DependentTemplateName *DTN = Name.getAsDependentTemplateName();
     DeclarationName DName;
     if (DTN->isIdentifier()) {
       DName = DeclarationNames.getIdentifier(DTN->getIdentifier());
       return DeclarationNameInfo(DName, NameLoc);
     } else {
       DName = DeclarationNames.getCXXOperatorName(DTN->getOperator());
       // DNInfo work in progress: FIXME: source locations?
       DeclarationNameLoc DNLoc =
           DeclarationNameLoc::makeCXXOperatorNameLoc(SourceRange());
       return DeclarationNameInfo(DName, NameLoc, DNLoc);
     }
   }
 
   case TemplateName::SubstTemplateTemplateParm: {
     SubstTemplateTemplateParmStorage *subst
       = Name.getAsSubstTemplateTemplateParm();
     return DeclarationNameInfo(subst->getParameter()->getDeclName(),
                                NameLoc);
   }
 
   case TemplateName::SubstTemplateTemplateParmPack: {
     SubstTemplateTemplateParmPackStorage *subst
       = Name.getAsSubstTemplateTemplateParmPack();
     return DeclarationNameInfo(subst->getParameterPack()->getDeclName(),
                                NameLoc);
   }
   }
 
   llvm_unreachable("bad template name kind!");
 }
 
 TemplateName
 ASTContext::getCanonicalTemplateName(const TemplateName &Name) const {
   switch (Name.getKind()) {
   case TemplateName::QualifiedTemplate:
   case TemplateName::Template: {
     TemplateDecl *Template = Name.getAsTemplateDecl();
     if (auto *TTP  = dyn_cast<TemplateTemplateParmDecl>(Template))
       Template = getCanonicalTemplateTemplateParmDecl(TTP);
 
     // The canonical template name is the canonical template declaration.
     return TemplateName(cast<TemplateDecl>(Template->getCanonicalDecl()));
   }
 
   case TemplateName::OverloadedTemplate:
   case TemplateName::AssumedTemplate:
     llvm_unreachable("cannot canonicalize unresolved template");
 
   case TemplateName::DependentTemplate: {
     DependentTemplateName *DTN = Name.getAsDependentTemplateName();
     assert(DTN && "Non-dependent template names must refer to template decls.");
     return DTN->CanonicalTemplateName;
   }
 
   case TemplateName::SubstTemplateTemplateParm: {
     SubstTemplateTemplateParmStorage *subst
       = Name.getAsSubstTemplateTemplateParm();
     return getCanonicalTemplateName(subst->getReplacement());
   }
 
   case TemplateName::SubstTemplateTemplateParmPack: {
     SubstTemplateTemplateParmPackStorage *subst
                                   = Name.getAsSubstTemplateTemplateParmPack();
     TemplateTemplateParmDecl *canonParameter
       = getCanonicalTemplateTemplateParmDecl(subst->getParameterPack());
     TemplateArgument canonArgPack
       = getCanonicalTemplateArgument(subst->getArgumentPack());
     return getSubstTemplateTemplateParmPack(canonParameter, canonArgPack);
   }
   }
 
   llvm_unreachable("bad template name!");
 }
 
 bool ASTContext::hasSameTemplateName(const TemplateName &X,
                                      const TemplateName &Y) const {
   return getCanonicalTemplateName(X).getAsVoidPointer() ==
          getCanonicalTemplateName(Y).getAsVoidPointer();
 }
 
 bool ASTContext::isSameTemplateParameter(const NamedDecl *X,
                                          const NamedDecl *Y) {
   if (X->getKind() != Y->getKind())
     return false;
 
   if (auto *TX = dyn_cast<TemplateTypeParmDecl>(X)) {
     auto *TY = cast<TemplateTypeParmDecl>(Y);
     if (TX->isParameterPack() != TY->isParameterPack())
       return false;
     if (TX->hasTypeConstraint() != TY->hasTypeConstraint())
       return false;
     const TypeConstraint *TXTC = TX->getTypeConstraint();
     const TypeConstraint *TYTC = TY->getTypeConstraint();
     if (!TXTC != !TYTC)
       return false;
     if (TXTC && TYTC) {
       auto *NCX = TXTC->getNamedConcept();
       auto *NCY = TYTC->getNamedConcept();
       if (!NCX || !NCY || !isSameEntity(NCX, NCY))
         return false;
       if (TXTC->hasExplicitTemplateArgs() != TYTC->hasExplicitTemplateArgs())
         return false;
       if (TXTC->hasExplicitTemplateArgs()) {
         auto *TXTCArgs = TXTC->getTemplateArgsAsWritten();
         auto *TYTCArgs = TYTC->getTemplateArgsAsWritten();
         if (TXTCArgs->NumTemplateArgs != TYTCArgs->NumTemplateArgs)
           return false;
         llvm::FoldingSetNodeID XID, YID;
         for (auto &ArgLoc : TXTCArgs->arguments())
           ArgLoc.getArgument().Profile(XID, X->getASTContext());
         for (auto &ArgLoc : TYTCArgs->arguments())
           ArgLoc.getArgument().Profile(YID, Y->getASTContext());
         if (XID != YID)
           return false;
       }
     }
     return true;
   }
 
   if (auto *TX = dyn_cast<NonTypeTemplateParmDecl>(X)) {
     auto *TY = cast<NonTypeTemplateParmDecl>(Y);
     return TX->isParameterPack() == TY->isParameterPack() &&
            TX->getASTContext().hasSameType(TX->getType(), TY->getType());
   }
 
   auto *TX = cast<TemplateTemplateParmDecl>(X);
   auto *TY = cast<TemplateTemplateParmDecl>(Y);
   return TX->isParameterPack() == TY->isParameterPack() &&
          isSameTemplateParameterList(TX->getTemplateParameters(),
                                      TY->getTemplateParameters());
 }
 
 bool ASTContext::isSameTemplateParameterList(const TemplateParameterList *X,
                                              const TemplateParameterList *Y) {
   if (X->size() != Y->size())
     return false;
 
   for (unsigned I = 0, N = X->size(); I != N; ++I)
     if (!isSameTemplateParameter(X->getParam(I), Y->getParam(I)))
       return false;
 
   const Expr *XRC = X->getRequiresClause();
   const Expr *YRC = Y->getRequiresClause();
   if (!XRC != !YRC)
     return false;
   if (XRC) {
     llvm::FoldingSetNodeID XRCID, YRCID;
     XRC->Profile(XRCID, *this, /*Canonical=*/true);
     YRC->Profile(YRCID, *this, /*Canonical=*/true);
     if (XRCID != YRCID)
       return false;
   }
 
   return true;
 }
 
 static NamespaceDecl *getNamespace(const NestedNameSpecifier *X) {
   if (auto *NS = X->getAsNamespace())
     return NS;
   if (auto *NAS = X->getAsNamespaceAlias())
     return NAS->getNamespace();
   return nullptr;
 }
 
 static bool isSameQualifier(const NestedNameSpecifier *X,
                             const NestedNameSpecifier *Y) {
   if (auto *NSX = getNamespace(X)) {
     auto *NSY = getNamespace(Y);
     if (!NSY || NSX->getCanonicalDecl() != NSY->getCanonicalDecl())
       return false;
   } else if (X->getKind() != Y->getKind())
     return false;
 
   // FIXME: For namespaces and types, we're permitted to check that the entity
   // is named via the same tokens. We should probably do so.
   switch (X->getKind()) {
   case NestedNameSpecifier::Identifier:
     if (X->getAsIdentifier() != Y->getAsIdentifier())
       return false;
     break;
   case NestedNameSpecifier::Namespace:
   case NestedNameSpecifier::NamespaceAlias:
     // We've already checked that we named the same namespace.
     break;
   case NestedNameSpecifier::TypeSpec:
   case NestedNameSpecifier::TypeSpecWithTemplate:
     if (X->getAsType()->getCanonicalTypeInternal() !=
         Y->getAsType()->getCanonicalTypeInternal())
       return false;
     break;
   case NestedNameSpecifier::Global:
   case NestedNameSpecifier::Super:
     return true;
   }
 
   // Recurse into earlier portion of NNS, if any.
   auto *PX = X->getPrefix();
   auto *PY = Y->getPrefix();
   if (PX && PY)
     return isSameQualifier(PX, PY);
   return !PX && !PY;
 }
 
 /// Determine whether the attributes we can overload on are identical for A and
 /// B. Will ignore any overloadable attrs represented in the type of A and B.
 static bool hasSameOverloadableAttrs(const FunctionDecl *A,
                                      const FunctionDecl *B) {
   // Note that pass_object_size attributes are represented in the function's
   // ExtParameterInfo, so we don't need to check them here.
 
   llvm::FoldingSetNodeID Cand1ID, Cand2ID;
   auto AEnableIfAttrs = A->specific_attrs<EnableIfAttr>();
   auto BEnableIfAttrs = B->specific_attrs<EnableIfAttr>();
 
   for (auto Pair : zip_longest(AEnableIfAttrs, BEnableIfAttrs)) {
     Optional<EnableIfAttr *> Cand1A = std::get<0>(Pair);
     Optional<EnableIfAttr *> Cand2A = std::get<1>(Pair);
 
     // Return false if the number of enable_if attributes is different.
     if (!Cand1A || !Cand2A)
       return false;
 
     Cand1ID.clear();
     Cand2ID.clear();
 
     (*Cand1A)->getCond()->Profile(Cand1ID, A->getASTContext(), true);
     (*Cand2A)->getCond()->Profile(Cand2ID, B->getASTContext(), true);
 
     // Return false if any of the enable_if expressions of A and B are
     // different.
     if (Cand1ID != Cand2ID)
       return false;
   }
   return true;
 }
 
 bool ASTContext::isSameEntity(const NamedDecl *X, const NamedDecl *Y) {
   if (X == Y)
     return true;
 
   if (X->getDeclName() != Y->getDeclName())
     return false;
 
   // Must be in the same context.
   //
   // Note that we can't use DeclContext::Equals here, because the DeclContexts
   // could be two different declarations of the same function. (We will fix the
   // semantic DC to refer to the primary definition after merging.)
   if (!declaresSameEntity(cast<Decl>(X->getDeclContext()->getRedeclContext()),
                           cast<Decl>(Y->getDeclContext()->getRedeclContext())))
     return false;
 
   // Two typedefs refer to the same entity if they have the same underlying
   // type.
   if (const auto *TypedefX = dyn_cast<TypedefNameDecl>(X))
     if (const auto *TypedefY = dyn_cast<TypedefNameDecl>(Y))
       return hasSameType(TypedefX->getUnderlyingType(),
                          TypedefY->getUnderlyingType());
 
   // Must have the same kind.
   if (X->getKind() != Y->getKind())
     return false;
 
   // Objective-C classes and protocols with the same name always match.
   if (isa<ObjCInterfaceDecl>(X) || isa<ObjCProtocolDecl>(X))
     return true;
 
   if (isa<ClassTemplateSpecializationDecl>(X)) {
     // No need to handle these here: we merge them when adding them to the
     // template.
     return false;
   }
 
   // Compatible tags match.
   if (const auto *TagX = dyn_cast<TagDecl>(X)) {
     const auto *TagY = cast<TagDecl>(Y);
     return (TagX->getTagKind() == TagY->getTagKind()) ||
            ((TagX->getTagKind() == TTK_Struct ||
              TagX->getTagKind() == TTK_Class ||
              TagX->getTagKind() == TTK_Interface) &&
             (TagY->getTagKind() == TTK_Struct ||
              TagY->getTagKind() == TTK_Class ||
              TagY->getTagKind() == TTK_Interface));
   }
 
   // Functions with the same type and linkage match.
   // FIXME: This needs to cope with merging of prototyped/non-prototyped
   // functions, etc.
   if (const auto *FuncX = dyn_cast<FunctionDecl>(X)) {
     const auto *FuncY = cast<FunctionDecl>(Y);
     if (const auto *CtorX = dyn_cast<CXXConstructorDecl>(X)) {
       const auto *CtorY = cast<CXXConstructorDecl>(Y);
       if (CtorX->getInheritedConstructor() &&
           !isSameEntity(CtorX->getInheritedConstructor().getConstructor(),
                         CtorY->getInheritedConstructor().getConstructor()))
         return false;
     }
 
     if (FuncX->isMultiVersion() != FuncY->isMultiVersion())
       return false;
 
     // Multiversioned functions with different feature strings are represented
     // as separate declarations.
     if (FuncX->isMultiVersion()) {
       const auto *TAX = FuncX->getAttr<TargetAttr>();
       const auto *TAY = FuncY->getAttr<TargetAttr>();
       assert(TAX && TAY && "Multiversion Function without target attribute");
 
       if (TAX->getFeaturesStr() != TAY->getFeaturesStr())
         return false;
     }
 
     const Expr *XRC = FuncX->getTrailingRequiresClause();
     const Expr *YRC = FuncY->getTrailingRequiresClause();
     if (!XRC != !YRC)
       return false;
     if (XRC) {
       llvm::FoldingSetNodeID XRCID, YRCID;
       XRC->Profile(XRCID, *this, /*Canonical=*/true);
       YRC->Profile(YRCID, *this, /*Canonical=*/true);
       if (XRCID != YRCID)
         return false;
     }
 
     auto GetTypeAsWritten = [](const FunctionDecl *FD) {
       // Map to the first declaration that we've already merged into this one.
       // The TSI of redeclarations might not match (due to calling conventions
       // being inherited onto the type but not the TSI), but the TSI type of
       // the first declaration of the function should match across modules.
       FD = FD->getCanonicalDecl();
       return FD->getTypeSourceInfo() ? FD->getTypeSourceInfo()->getType()
                                      : FD->getType();
     };
     QualType XT = GetTypeAsWritten(FuncX), YT = GetTypeAsWritten(FuncY);
     if (!hasSameType(XT, YT)) {
       // We can get functions with different types on the redecl chain in C++17
       // if they have differing exception specifications and at least one of
       // the excpetion specs is unresolved.
       auto *XFPT = XT->getAs<FunctionProtoType>();
       auto *YFPT = YT->getAs<FunctionProtoType>();
       if (getLangOpts().CPlusPlus17 && XFPT && YFPT &&
           (isUnresolvedExceptionSpec(XFPT->getExceptionSpecType()) ||
            isUnresolvedExceptionSpec(YFPT->getExceptionSpecType())) &&
           // FIXME: We could make isSameEntity const after we make
           // hasSameFunctionTypeIgnoringExceptionSpec const.
           hasSameFunctionTypeIgnoringExceptionSpec(XT, YT))
         return true;
       return false;
     }
 
     return FuncX->getLinkageInternal() == FuncY->getLinkageInternal() &&
            hasSameOverloadableAttrs(FuncX, FuncY);
   }
 
   // Variables with the same type and linkage match.
   if (const auto *VarX = dyn_cast<VarDecl>(X)) {
     const auto *VarY = cast<VarDecl>(Y);
     if (VarX->getLinkageInternal() == VarY->getLinkageInternal()) {
       if (hasSameType(VarX->getType(), VarY->getType()))
         return true;
 
       // We can get decls with different types on the redecl chain. Eg.
       // template <typename T> struct S { static T Var[]; }; // #1
       // template <typename T> T S<T>::Var[sizeof(T)]; // #2
       // Only? happens when completing an incomplete array type. In this case
       // when comparing #1 and #2 we should go through their element type.
       const ArrayType *VarXTy = getAsArrayType(VarX->getType());
       const ArrayType *VarYTy = getAsArrayType(VarY->getType());
       if (!VarXTy || !VarYTy)
         return false;
       if (VarXTy->isIncompleteArrayType() || VarYTy->isIncompleteArrayType())
         return hasSameType(VarXTy->getElementType(), VarYTy->getElementType());
     }
     return false;
   }
 
   // Namespaces with the same name and inlinedness match.
   if (const auto *NamespaceX = dyn_cast<NamespaceDecl>(X)) {
     const auto *NamespaceY = cast<NamespaceDecl>(Y);
     return NamespaceX->isInline() == NamespaceY->isInline();
   }
 
   // Identical template names and kinds match if their template parameter lists
   // and patterns match.
   if (const auto *TemplateX = dyn_cast<TemplateDecl>(X)) {
     const auto *TemplateY = cast<TemplateDecl>(Y);
     return isSameEntity(TemplateX->getTemplatedDecl(),
                         TemplateY->getTemplatedDecl()) &&
            isSameTemplateParameterList(TemplateX->getTemplateParameters(),
                                        TemplateY->getTemplateParameters());
   }
 
   // Fields with the same name and the same type match.
   if (const auto *FDX = dyn_cast<FieldDecl>(X)) {
     const auto *FDY = cast<FieldDecl>(Y);
     // FIXME: Also check the bitwidth is odr-equivalent, if any.
     return hasSameType(FDX->getType(), FDY->getType());
   }
 
   // Indirect fields with the same target field match.
   if (const auto *IFDX = dyn_cast<IndirectFieldDecl>(X)) {
     const auto *IFDY = cast<IndirectFieldDecl>(Y);
     return IFDX->getAnonField()->getCanonicalDecl() ==
            IFDY->getAnonField()->getCanonicalDecl();
   }
 
   // Enumerators with the same name match.
   if (isa<EnumConstantDecl>(X))
     // FIXME: Also check the value is odr-equivalent.
     return true;
 
   // Using shadow declarations with the same target match.
   if (const auto *USX = dyn_cast<UsingShadowDecl>(X)) {
     const auto *USY = cast<UsingShadowDecl>(Y);
     return USX->getTargetDecl() == USY->getTargetDecl();
   }
 
   // Using declarations with the same qualifier match. (We already know that
   // the name matches.)
   if (const auto *UX = dyn_cast<UsingDecl>(X)) {
     const auto *UY = cast<UsingDecl>(Y);
     return isSameQualifier(UX->getQualifier(), UY->getQualifier()) &&
            UX->hasTypename() == UY->hasTypename() &&
            UX->isAccessDeclaration() == UY->isAccessDeclaration();
   }
   if (const auto *UX = dyn_cast<UnresolvedUsingValueDecl>(X)) {
     const auto *UY = cast<UnresolvedUsingValueDecl>(Y);
     return isSameQualifier(UX->getQualifier(), UY->getQualifier()) &&
            UX->isAccessDeclaration() == UY->isAccessDeclaration();
   }
   if (const auto *UX = dyn_cast<UnresolvedUsingTypenameDecl>(X)) {
     return isSameQualifier(
         UX->getQualifier(),
         cast<UnresolvedUsingTypenameDecl>(Y)->getQualifier());
   }
 
   // Using-pack declarations are only created by instantiation, and match if
   // they're instantiated from matching UnresolvedUsing...Decls.
   if (const auto *UX = dyn_cast<UsingPackDecl>(X)) {
     return declaresSameEntity(
         UX->getInstantiatedFromUsingDecl(),
         cast<UsingPackDecl>(Y)->getInstantiatedFromUsingDecl());
   }
 
   // Namespace alias definitions with the same target match.
   if (const auto *NAX = dyn_cast<NamespaceAliasDecl>(X)) {
     const auto *NAY = cast<NamespaceAliasDecl>(Y);
     return NAX->getNamespace()->Equals(NAY->getNamespace());
   }
 
   return false;
 }
 
 TemplateArgument
 ASTContext::getCanonicalTemplateArgument(const TemplateArgument &Arg) const {
   switch (Arg.getKind()) {
     case TemplateArgument::Null:
       return Arg;
 
     case TemplateArgument::Expression:
       return Arg;
 
     case TemplateArgument::Declaration: {
       auto *D = cast<ValueDecl>(Arg.getAsDecl()->getCanonicalDecl());
       return TemplateArgument(D, Arg.getParamTypeForDecl());
     }
 
     case TemplateArgument::NullPtr:
       return TemplateArgument(getCanonicalType(Arg.getNullPtrType()),
                               /*isNullPtr*/true);
 
     case TemplateArgument::Template:
       return TemplateArgument(getCanonicalTemplateName(Arg.getAsTemplate()));
 
     case TemplateArgument::TemplateExpansion:
       return TemplateArgument(getCanonicalTemplateName(
                                          Arg.getAsTemplateOrTemplatePattern()),
                               Arg.getNumTemplateExpansions());
 
     case TemplateArgument::Integral:
       return TemplateArgument(Arg, getCanonicalType(Arg.getIntegralType()));
 
     case TemplateArgument::Type:
       return TemplateArgument(getCanonicalType(Arg.getAsType()));
 
     case TemplateArgument::Pack: {
       if (Arg.pack_size() == 0)
         return Arg;
 
       auto *CanonArgs = new (*this) TemplateArgument[Arg.pack_size()];
       unsigned Idx = 0;
       for (TemplateArgument::pack_iterator A = Arg.pack_begin(),
                                         AEnd = Arg.pack_end();
            A != AEnd; (void)++A, ++Idx)
         CanonArgs[Idx] = getCanonicalTemplateArgument(*A);
 
       return TemplateArgument(llvm::makeArrayRef(CanonArgs, Arg.pack_size()));
     }
   }
 
   // Silence GCC warning
   llvm_unreachable("Unhandled template argument kind");
 }
 
 NestedNameSpecifier *
 ASTContext::getCanonicalNestedNameSpecifier(NestedNameSpecifier *NNS) const {
   if (!NNS)
     return nullptr;
 
   switch (NNS->getKind()) {
   case NestedNameSpecifier::Identifier:
     // Canonicalize the prefix but keep the identifier the same.
     return NestedNameSpecifier::Create(*this,
                          getCanonicalNestedNameSpecifier(NNS->getPrefix()),
                                        NNS->getAsIdentifier());
 
   case NestedNameSpecifier::Namespace:
     // A namespace is canonical; build a nested-name-specifier with
     // this namespace and no prefix.
     return NestedNameSpecifier::Create(*this, nullptr,
                                  NNS->getAsNamespace()->getOriginalNamespace());
 
   case NestedNameSpecifier::NamespaceAlias:
     // A namespace is canonical; build a nested-name-specifier with
     // this namespace and no prefix.
     return NestedNameSpecifier::Create(*this, nullptr,
                                     NNS->getAsNamespaceAlias()->getNamespace()
                                                       ->getOriginalNamespace());
 
   // The difference between TypeSpec and TypeSpecWithTemplate is that the
   // latter will have the 'template' keyword when printed.
   case NestedNameSpecifier::TypeSpec:
   case NestedNameSpecifier::TypeSpecWithTemplate: {
     const Type *T = getCanonicalType(NNS->getAsType());
 
     // If we have some kind of dependent-named type (e.g., "typename T::type"),
     // break it apart into its prefix and identifier, then reconsititute those
     // as the canonical nested-name-specifier. This is required to canonicalize
     // a dependent nested-name-specifier involving typedefs of dependent-name
     // types, e.g.,
     //   typedef typename T::type T1;
     //   typedef typename T1::type T2;
     if (const auto *DNT = T->getAs<DependentNameType>())
       return NestedNameSpecifier::Create(
           *this, DNT->getQualifier(),
           const_cast<IdentifierInfo *>(DNT->getIdentifier()));
     if (const auto *DTST = T->getAs<DependentTemplateSpecializationType>())
       return NestedNameSpecifier::Create(*this, DTST->getQualifier(), true,
                                          const_cast<Type *>(T));
 
     // TODO: Set 'Template' parameter to true for other template types.
     return NestedNameSpecifier::Create(*this, nullptr, false,
                                        const_cast<Type *>(T));
   }
 
   case NestedNameSpecifier::Global:
   case NestedNameSpecifier::Super:
     // The global specifier and __super specifer are canonical and unique.
     return NNS;
   }
 
   llvm_unreachable("Invalid NestedNameSpecifier::Kind!");
 }
 
 const ArrayType *ASTContext::getAsArrayType(QualType T) const {
   // Handle the non-qualified case efficiently.
   if (!T.hasLocalQualifiers()) {
     // Handle the common positive case fast.
     if (const auto *AT = dyn_cast<ArrayType>(T))
       return AT;
   }
 
   // Handle the common negative case fast.
   if (!isa<ArrayType>(T.getCanonicalType()))
     return nullptr;
 
   // Apply any qualifiers from the array type to the element type.  This
   // implements C99 6.7.3p8: "If the specification of an array type includes
   // any type qualifiers, the element type is so qualified, not the array type."
 
   // If we get here, we either have type qualifiers on the type, or we have
   // sugar such as a typedef in the way.  If we have type qualifiers on the type
   // we must propagate them down into the element type.
 
   SplitQualType split = T.getSplitDesugaredType();
   Qualifiers qs = split.Quals;
 
   // If we have a simple case, just return now.
   const auto *ATy = dyn_cast<ArrayType>(split.Ty);
   if (!ATy || qs.empty())
     return ATy;
 
   // Otherwise, we have an array and we have qualifiers on it.  Push the
   // qualifiers into the array element type and return a new array type.
   QualType NewEltTy = getQualifiedType(ATy->getElementType(), qs);
 
   if (const auto *CAT = dyn_cast<ConstantArrayType>(ATy))
     return cast<ArrayType>(getConstantArrayType(NewEltTy, CAT->getSize(),
                                                 CAT->getSizeExpr(),
                                                 CAT->getSizeModifier(),
                                            CAT->getIndexTypeCVRQualifiers()));
   if (const auto *IAT = dyn_cast<IncompleteArrayType>(ATy))
     return cast<ArrayType>(getIncompleteArrayType(NewEltTy,
                                                   IAT->getSizeModifier(),
                                            IAT->getIndexTypeCVRQualifiers()));
 
   if (const auto *DSAT = dyn_cast<DependentSizedArrayType>(ATy))
     return cast<ArrayType>(
                      getDependentSizedArrayType(NewEltTy,
                                                 DSAT->getSizeExpr(),
                                                 DSAT->getSizeModifier(),
                                               DSAT->getIndexTypeCVRQualifiers(),
                                                 DSAT->getBracketsRange()));
 
   const auto *VAT = cast<VariableArrayType>(ATy);
   return cast<ArrayType>(getVariableArrayType(NewEltTy,
                                               VAT->getSizeExpr(),
                                               VAT->getSizeModifier(),
                                               VAT->getIndexTypeCVRQualifiers(),
                                               VAT->getBracketsRange()));
 }
 
 QualType ASTContext::getAdjustedParameterType(QualType T) const {
   if (T->isArrayType() || T->isFunctionType())
     return getDecayedType(T);
   return T;
 }
 
 QualType ASTContext::getSignatureParameterType(QualType T) const {
   T = getVariableArrayDecayedType(T);
   T = getAdjustedParameterType(T);
   return T.getUnqualifiedType();
 }
 
 QualType ASTContext::getExceptionObjectType(QualType T) const {
   // C++ [except.throw]p3:
   //   A throw-expression initializes a temporary object, called the exception
   //   object, the type of which is determined by removing any top-level
   //   cv-qualifiers from the static type of the operand of throw and adjusting
   //   the type from "array of T" or "function returning T" to "pointer to T"
   //   or "pointer to function returning T", [...]
   T = getVariableArrayDecayedType(T);
   if (T->isArrayType() || T->isFunctionType())
     T = getDecayedType(T);
   return T.getUnqualifiedType();
 }
 
 /// getArrayDecayedType - Return the properly qualified result of decaying the
 /// specified array type to a pointer.  This operation is non-trivial when
 /// handling typedefs etc.  The canonical type of "T" must be an array type,
 /// this returns a pointer to a properly qualified element of the array.
 ///
 /// See C99 6.7.5.3p7 and C99 6.3.2.1p3.
 QualType ASTContext::getArrayDecayedType(QualType Ty) const {
   // Get the element type with 'getAsArrayType' so that we don't lose any
   // typedefs in the element type of the array.  This also handles propagation
   // of type qualifiers from the array type into the element type if present
   // (C99 6.7.3p8).
   const ArrayType *PrettyArrayType = getAsArrayType(Ty);
   assert(PrettyArrayType && "Not an array type!");
 
   QualType PtrTy = getPointerType(PrettyArrayType->getElementType());
 
   // int x[restrict 4] ->  int *restrict
   QualType Result = getQualifiedType(PtrTy,
                                      PrettyArrayType->getIndexTypeQualifiers());
 
   // int x[_Nullable] -> int * _Nullable
   if (auto Nullability = Ty->getNullability(*this)) {
     Result = const_cast<ASTContext *>(this)->getAttributedType(
         AttributedType::getNullabilityAttrKind(*Nullability), Result, Result);
   }
   return Result;
 }
 
 QualType ASTContext::getBaseElementType(const ArrayType *array) const {
   return getBaseElementType(array->getElementType());
 }
 
 QualType ASTContext::getBaseElementType(QualType type) const {
   Qualifiers qs;
   while (true) {
     SplitQualType split = type.getSplitDesugaredType();
     const ArrayType *array = split.Ty->getAsArrayTypeUnsafe();
     if (!array) break;
 
     type = array->getElementType();
     qs.addConsistentQualifiers(split.Quals);
   }
 
   return getQualifiedType(type, qs);
 }
 
 /// getConstantArrayElementCount - Returns number of constant array elements.
 uint64_t
 ASTContext::getConstantArrayElementCount(const ConstantArrayType *CA)  const {
   uint64_t ElementCount = 1;
   do {
     ElementCount *= CA->getSize().getZExtValue();
     CA = dyn_cast_or_null<ConstantArrayType>(
       CA->getElementType()->getAsArrayTypeUnsafe());
   } while (CA);
   return ElementCount;
 }
 
 /// getFloatingRank - Return a relative rank for floating point types.
 /// This routine will assert if passed a built-in type that isn't a float.
 static FloatingRank getFloatingRank(QualType T) {
   if (const auto *CT = T->getAs<ComplexType>())
     return getFloatingRank(CT->getElementType());
 
   switch (T->castAs<BuiltinType>()->getKind()) {
   default: llvm_unreachable("getFloatingRank(): not a floating type");
   case BuiltinType::Float16:    return Float16Rank;
   case BuiltinType::Half:       return HalfRank;
   case BuiltinType::Float:      return FloatRank;
   case BuiltinType::Double:     return DoubleRank;
   case BuiltinType::LongDouble: return LongDoubleRank;
   case BuiltinType::Float128:   return Float128Rank;
   case BuiltinType::BFloat16:   return BFloat16Rank;
   case BuiltinType::Ibm128:     return Ibm128Rank;
   }
 }
 
 /// getFloatingTypeOfSizeWithinDomain - Returns a real floating
 /// point or a complex type (based on typeDomain/typeSize).
 /// 'typeDomain' is a real floating point or complex type.
 /// 'typeSize' is a real floating point or complex type.
 QualType ASTContext::getFloatingTypeOfSizeWithinDomain(QualType Size,
                                                        QualType Domain) const {
   FloatingRank EltRank = getFloatingRank(Size);
   if (Domain->isComplexType()) {
     switch (EltRank) {
     case BFloat16Rank: llvm_unreachable("Complex bfloat16 is not supported");
     case Float16Rank:
     case HalfRank: llvm_unreachable("Complex half is not supported");
     case Ibm128Rank:     return getComplexType(Ibm128Ty);
     case FloatRank:      return getComplexType(FloatTy);
     case DoubleRank:     return getComplexType(DoubleTy);
     case LongDoubleRank: return getComplexType(LongDoubleTy);
     case Float128Rank:   return getComplexType(Float128Ty);
     }
   }
 
   assert(Domain->isRealFloatingType() && "Unknown domain!");
   switch (EltRank) {
   case Float16Rank:    return HalfTy;
   case BFloat16Rank:   return BFloat16Ty;
   case HalfRank:       return HalfTy;
   case FloatRank:      return FloatTy;
   case DoubleRank:     return DoubleTy;
   case LongDoubleRank: return LongDoubleTy;
   case Float128Rank:   return Float128Ty;
   case Ibm128Rank:
     return Ibm128Ty;
   }
   llvm_unreachable("getFloatingRank(): illegal value for rank");
 }
 
 /// getFloatingTypeOrder - Compare the rank of the two specified floating
 /// point types, ignoring the domain of the type (i.e. 'double' ==
 /// '_Complex double').  If LHS > RHS, return 1.  If LHS == RHS, return 0. If
 /// LHS < RHS, return -1.
 int ASTContext::getFloatingTypeOrder(QualType LHS, QualType RHS) const {
   FloatingRank LHSR = getFloatingRank(LHS);
   FloatingRank RHSR = getFloatingRank(RHS);
 
   if (LHSR == RHSR)
     return 0;
   if (LHSR > RHSR)
     return 1;
   return -1;
 }
 
 int ASTContext::getFloatingTypeSemanticOrder(QualType LHS, QualType RHS) const {
   if (&getFloatTypeSemantics(LHS) == &getFloatTypeSemantics(RHS))
     return 0;
   return getFloatingTypeOrder(LHS, RHS);
 }
 
 /// getIntegerRank - Return an integer conversion rank (C99 6.3.1.1p1). This
 /// routine will assert if passed a built-in type that isn't an integer or enum,
 /// or if it is not canonicalized.
 unsigned ASTContext::getIntegerRank(const Type *T) const {
   assert(T->isCanonicalUnqualified() && "T should be canonicalized");
 
   // Results in this 'losing' to any type of the same size, but winning if
   // larger.
   if (const auto *EIT = dyn_cast<BitIntType>(T))
     return 0 + (EIT->getNumBits() << 3);
 
   switch (cast<BuiltinType>(T)->getKind()) {
   default: llvm_unreachable("getIntegerRank(): not a built-in integer");
   case BuiltinType::Bool:
     return 1 + (getIntWidth(BoolTy) << 3);
   case BuiltinType::Char_S:
   case BuiltinType::Char_U:
   case BuiltinType::SChar:
   case BuiltinType::UChar:
     return 2 + (getIntWidth(CharTy) << 3);
   case BuiltinType::Short:
   case BuiltinType::UShort:
     return 3 + (getIntWidth(ShortTy) << 3);
   case BuiltinType::Int:
   case BuiltinType::UInt:
     return 4 + (getIntWidth(IntTy) << 3);
   case BuiltinType::Long:
   case BuiltinType::ULong:
     return 5 + (getIntWidth(LongTy) << 3);
   case BuiltinType::LongLong:
   case BuiltinType::ULongLong:
     return 6 + (getIntWidth(LongLongTy) << 3);
   case BuiltinType::Int128:
   case BuiltinType::UInt128:
     return 7 + (getIntWidth(Int128Ty) << 3);
   }
 }
 
 /// Whether this is a promotable bitfield reference according
 /// to C99 6.3.1.1p2, bullet 2 (and GCC extensions).
 ///
 /// \returns the type this bit-field will promote to, or NULL if no
 /// promotion occurs.
 QualType ASTContext::isPromotableBitField(Expr *E) const {
   if (E->isTypeDependent() || E->isValueDependent())
     return {};
 
   // C++ [conv.prom]p5:
   //    If the bit-field has an enumerated type, it is treated as any other
   //    value of that type for promotion purposes.
   if (getLangOpts().CPlusPlus && E->getType()->isEnumeralType())
     return {};
 
   // FIXME: We should not do this unless E->refersToBitField() is true. This
   // matters in C where getSourceBitField() will find bit-fields for various
   // cases where the source expression is not a bit-field designator.
 
   FieldDecl *Field = E->getSourceBitField(); // FIXME: conditional bit-fields?
   if (!Field)
     return {};
 
   QualType FT = Field->getType();
 
   uint64_t BitWidth = Field->getBitWidthValue(*this);
   uint64_t IntSize = getTypeSize(IntTy);
   // C++ [conv.prom]p5:
   //   A prvalue for an integral bit-field can be converted to a prvalue of type
   //   int if int can represent all the values of the bit-field; otherwise, it
   //   can be converted to unsigned int if unsigned int can represent all the
   //   values of the bit-field. If the bit-field is larger yet, no integral
   //   promotion applies to it.
   // C11 6.3.1.1/2:
   //   [For a bit-field of type _Bool, int, signed int, or unsigned int:]
   //   If an int can represent all values of the original type (as restricted by
   //   the width, for a bit-field), the value is converted to an int; otherwise,
   //   it is converted to an unsigned int.
   //
   // FIXME: C does not permit promotion of a 'long : 3' bitfield to int.
   //        We perform that promotion here to match GCC and C++.
   // FIXME: C does not permit promotion of an enum bit-field whose rank is
   //        greater than that of 'int'. We perform that promotion to match GCC.
   if (BitWidth < IntSize)
     return IntTy;
 
   if (BitWidth == IntSize)
     return FT->isSignedIntegerType() ? IntTy : UnsignedIntTy;
 
   // Bit-fields wider than int are not subject to promotions, and therefore act
   // like the base type. GCC has some weird bugs in this area that we
   // deliberately do not follow (GCC follows a pre-standard resolution to
   // C's DR315 which treats bit-width as being part of the type, and this leaks
   // into their semantics in some cases).
   return {};
 }
 
 /// getPromotedIntegerType - Returns the type that Promotable will
 /// promote to: C99 6.3.1.1p2, assuming that Promotable is a promotable
 /// integer type.
 QualType ASTContext::getPromotedIntegerType(QualType Promotable) const {
   assert(!Promotable.isNull());
   assert(Promotable->isPromotableIntegerType());
   if (const auto *ET = Promotable->getAs<EnumType>())
     return ET->getDecl()->getPromotionType();
 
   if (const auto *BT = Promotable->getAs<BuiltinType>()) {
     // C++ [conv.prom]: A prvalue of type char16_t, char32_t, or wchar_t
     // (3.9.1) can be converted to a prvalue of the first of the following
     // types that can represent all the values of its underlying type:
     // int, unsigned int, long int, unsigned long int, long long int, or
     // unsigned long long int [...]
     // FIXME: Is there some better way to compute this?
     if (BT->getKind() == BuiltinType::WChar_S ||
         BT->getKind() == BuiltinType::WChar_U ||
         BT->getKind() == BuiltinType::Char8 ||
         BT->getKind() == BuiltinType::Char16 ||
         BT->getKind() == BuiltinType::Char32) {
       bool FromIsSigned = BT->getKind() == BuiltinType::WChar_S;
       uint64_t FromSize = getTypeSize(BT);
       QualType PromoteTypes[] = { IntTy, UnsignedIntTy, LongTy, UnsignedLongTy,
                                   LongLongTy, UnsignedLongLongTy };
       for (size_t Idx = 0; Idx < llvm::array_lengthof(PromoteTypes); ++Idx) {
         uint64_t ToSize = getTypeSize(PromoteTypes[Idx]);
         if (FromSize < ToSize ||
             (FromSize == ToSize &&
              FromIsSigned == PromoteTypes[Idx]->isSignedIntegerType()))
           return PromoteTypes[Idx];
       }
       llvm_unreachable("char type should fit into long long");
     }
   }
 
   // At this point, we should have a signed or unsigned integer type.
   if (Promotable->isSignedIntegerType())
     return IntTy;
   uint64_t PromotableSize = getIntWidth(Promotable);
   uint64_t IntSize = getIntWidth(IntTy);
   assert(Promotable->isUnsignedIntegerType() && PromotableSize <= IntSize);
   return (PromotableSize != IntSize) ? IntTy : UnsignedIntTy;
 }
 
 /// Recurses in pointer/array types until it finds an objc retainable
 /// type and returns its ownership.
 Qualifiers::ObjCLifetime ASTContext::getInnerObjCOwnership(QualType T) const {
   while (!T.isNull()) {
     if (T.getObjCLifetime() != Qualifiers::OCL_None)
       return T.getObjCLifetime();
     if (T->isArrayType())
       T = getBaseElementType(T);
     else if (const auto *PT = T->getAs<PointerType>())
       T = PT->getPointeeType();
     else if (const auto *RT = T->getAs<ReferenceType>())
       T = RT->getPointeeType();
     else
       break;
   }
 
   return Qualifiers::OCL_None;
 }
 
 static const Type *getIntegerTypeForEnum(const EnumType *ET) {
   // Incomplete enum types are not treated as integer types.
   // FIXME: In C++, enum types are never integer types.
   if (ET->getDecl()->isComplete() && !ET->getDecl()->isScoped())
     return ET->getDecl()->getIntegerType().getTypePtr();
   return nullptr;
 }
 
 /// getIntegerTypeOrder - Returns the highest ranked integer type:
 /// C99 6.3.1.8p1.  If LHS > RHS, return 1.  If LHS == RHS, return 0. If
 /// LHS < RHS, return -1.
 int ASTContext::getIntegerTypeOrder(QualType LHS, QualType RHS) const {
   const Type *LHSC = getCanonicalType(LHS).getTypePtr();
   const Type *RHSC = getCanonicalType(RHS).getTypePtr();
 
   // Unwrap enums to their underlying type.
   if (const auto *ET = dyn_cast<EnumType>(LHSC))
     LHSC = getIntegerTypeForEnum(ET);
   if (const auto *ET = dyn_cast<EnumType>(RHSC))
     RHSC = getIntegerTypeForEnum(ET);
 
   if (LHSC == RHSC) return 0;
 
   bool LHSUnsigned = LHSC->isUnsignedIntegerType();
   bool RHSUnsigned = RHSC->isUnsignedIntegerType();
 
   unsigned LHSRank = getIntegerRank(LHSC);
   unsigned RHSRank = getIntegerRank(RHSC);
 
   if (LHSUnsigned == RHSUnsigned) {  // Both signed or both unsigned.
     if (LHSRank == RHSRank) return 0;
     return LHSRank > RHSRank ? 1 : -1;
   }
 
   // Otherwise, the LHS is signed and the RHS is unsigned or visa versa.
   if (LHSUnsigned) {
     // If the unsigned [LHS] type is larger, return it.
     if (LHSRank >= RHSRank)
       return 1;
 
     // If the signed type can represent all values of the unsigned type, it
     // wins.  Because we are dealing with 2's complement and types that are
     // powers of two larger than each other, this is always safe.
     return -1;
   }
 
   // If the unsigned [RHS] type is larger, return it.
   if (RHSRank >= LHSRank)
     return -1;
 
   // If the signed type can represent all values of the unsigned type, it
   // wins.  Because we are dealing with 2's complement and types that are
   // powers of two larger than each other, this is always safe.
   return 1;
 }
 
 TypedefDecl *ASTContext::getCFConstantStringDecl() const {
   if (CFConstantStringTypeDecl)
     return CFConstantStringTypeDecl;
 
   assert(!CFConstantStringTagDecl &&
          "tag and typedef should be initialized together");
   CFConstantStringTagDecl = buildImplicitRecord("__NSConstantString_tag");
   CFConstantStringTagDecl->startDefinition();
 
   struct {
     QualType Type;
     const char *Name;
   } Fields[5];
   unsigned Count = 0;
 
   /// Objective-C ABI
   ///
   ///    typedef struct __NSConstantString_tag {
   ///      const int *isa;
   ///      int flags;
   ///      const char *str;
   ///      long length;
   ///    } __NSConstantString;
   ///
   /// Swift ABI (4.1, 4.2)
   ///
   ///    typedef struct __NSConstantString_tag {
   ///      uintptr_t _cfisa;
   ///      uintptr_t _swift_rc;
   ///      _Atomic(uint64_t) _cfinfoa;
   ///      const char *_ptr;
   ///      uint32_t _length;
   ///    } __NSConstantString;
   ///
   /// Swift ABI (5.0)
   ///
   ///    typedef struct __NSConstantString_tag {
   ///      uintptr_t _cfisa;
   ///      uintptr_t _swift_rc;
   ///      _Atomic(uint64_t) _cfinfoa;
   ///      const char *_ptr;
   ///      uintptr_t _length;
   ///    } __NSConstantString;
 
   const auto CFRuntime = getLangOpts().CFRuntime;
   if (static_cast<unsigned>(CFRuntime) <
       static_cast<unsigned>(LangOptions::CoreFoundationABI::Swift)) {
     Fields[Count++] = { getPointerType(IntTy.withConst()), "isa" };
     Fields[Count++] = { IntTy, "flags" };
     Fields[Count++] = { getPointerType(CharTy.withConst()), "str" };
     Fields[Count++] = { LongTy, "length" };
   } else {
     Fields[Count++] = { getUIntPtrType(), "_cfisa" };
     Fields[Count++] = { getUIntPtrType(), "_swift_rc" };
     Fields[Count++] = { getFromTargetType(Target->getUInt64Type()), "_swift_rc" };
     Fields[Count++] = { getPointerType(CharTy.withConst()), "_ptr" };
     if (CFRuntime == LangOptions::CoreFoundationABI::Swift4_1 ||
         CFRuntime == LangOptions::CoreFoundationABI::Swift4_2)
       Fields[Count++] = { IntTy, "_ptr" };
     else
       Fields[Count++] = { getUIntPtrType(), "_ptr" };
   }
 
   // Create fields
   for (unsigned i = 0; i < Count; ++i) {
     FieldDecl *Field =
         FieldDecl::Create(*this, CFConstantStringTagDecl, SourceLocation(),
                           SourceLocation(), &Idents.get(Fields[i].Name),
                           Fields[i].Type, /*TInfo=*/nullptr,
                           /*BitWidth=*/nullptr, /*Mutable=*/false, ICIS_NoInit);
     Field->setAccess(AS_public);
     CFConstantStringTagDecl->addDecl(Field);
   }
 
   CFConstantStringTagDecl->completeDefinition();
   // This type is designed to be compatible with NSConstantString, but cannot
   // use the same name, since NSConstantString is an interface.
   auto tagType = getTagDeclType(CFConstantStringTagDecl);
   CFConstantStringTypeDecl =
       buildImplicitTypedef(tagType, "__NSConstantString");
 
   return CFConstantStringTypeDecl;
 }
 
 RecordDecl *ASTContext::getCFConstantStringTagDecl() const {
   if (!CFConstantStringTagDecl)
     getCFConstantStringDecl(); // Build the tag and the typedef.
   return CFConstantStringTagDecl;
 }
 
 // getCFConstantStringType - Return the type used for constant CFStrings.
 QualType ASTContext::getCFConstantStringType() const {
   return getTypedefType(getCFConstantStringDecl());
 }
 
 QualType ASTContext::getObjCSuperType() const {
   if (ObjCSuperType.isNull()) {
     RecordDecl *ObjCSuperTypeDecl = buildImplicitRecord("objc_super");
     getTranslationUnitDecl()->addDecl(ObjCSuperTypeDecl);
     ObjCSuperType = getTagDeclType(ObjCSuperTypeDecl);
   }
   return ObjCSuperType;
 }
 
 void ASTContext::setCFConstantStringType(QualType T) {
   const auto *TD = T->castAs<TypedefType>();
   CFConstantStringTypeDecl = cast<TypedefDecl>(TD->getDecl());
   const auto *TagType =
       CFConstantStringTypeDecl->getUnderlyingType()->castAs<RecordType>();
   CFConstantStringTagDecl = TagType->getDecl();
 }
 
 QualType ASTContext::getBlockDescriptorType() const {
   if (BlockDescriptorType)
     return getTagDeclType(BlockDescriptorType);
 
   RecordDecl *RD;
   // FIXME: Needs the FlagAppleBlock bit.
   RD = buildImplicitRecord("__block_descriptor");
   RD->startDefinition();
 
   QualType FieldTypes[] = {
     UnsignedLongTy,
     UnsignedLongTy,
   };
 
   static const char *const FieldNames[] = {
     "reserved",
     "Size"
   };
 
   for (size_t i = 0; i < 2; ++i) {
     FieldDecl *Field = FieldDecl::Create(
         *this, RD, SourceLocation(), SourceLocation(),
         &Idents.get(FieldNames[i]), FieldTypes[i], /*TInfo=*/nullptr,
         /*BitWidth=*/nullptr, /*Mutable=*/false, ICIS_NoInit);
     Field->setAccess(AS_public);
     RD->addDecl(Field);
   }
 
   RD->completeDefinition();
 
   BlockDescriptorType = RD;
 
   return getTagDeclType(BlockDescriptorType);
 }
 
 QualType ASTContext::getBlockDescriptorExtendedType() const {
   if (BlockDescriptorExtendedType)
     return getTagDeclType(BlockDescriptorExtendedType);
 
   RecordDecl *RD;
   // FIXME: Needs the FlagAppleBlock bit.
   RD = buildImplicitRecord("__block_descriptor_withcopydispose");
   RD->startDefinition();
 
   QualType FieldTypes[] = {
     UnsignedLongTy,
     UnsignedLongTy,
     getPointerType(VoidPtrTy),
     getPointerType(VoidPtrTy)
   };
 
   static const char *const FieldNames[] = {
     "reserved",
     "Size",
     "CopyFuncPtr",
     "DestroyFuncPtr"
   };
 
   for (size_t i = 0; i < 4; ++i) {
     FieldDecl *Field = FieldDecl::Create(
         *this, RD, SourceLocation(), SourceLocation(),
         &Idents.get(FieldNames[i]), FieldTypes[i], /*TInfo=*/nullptr,
         /*BitWidth=*/nullptr,
         /*Mutable=*/false, ICIS_NoInit);
     Field->setAccess(AS_public);
     RD->addDecl(Field);
   }
 
   RD->completeDefinition();
 
   BlockDescriptorExtendedType = RD;
   return getTagDeclType(BlockDescriptorExtendedType);
 }
 
 OpenCLTypeKind ASTContext::getOpenCLTypeKind(const Type *T) const {
   const auto *BT = dyn_cast<BuiltinType>(T);
 
   if (!BT) {
     if (isa<PipeType>(T))
       return OCLTK_Pipe;
 
     return OCLTK_Default;
   }
 
   switch (BT->getKind()) {
 #define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix)                   \
   case BuiltinType::Id:                                                        \
     return OCLTK_Image;
 #include "clang/Basic/OpenCLImageTypes.def"
 
   case BuiltinType::OCLClkEvent:
     return OCLTK_ClkEvent;
 
   case BuiltinType::OCLEvent:
     return OCLTK_Event;
 
   case BuiltinType::OCLQueue:
     return OCLTK_Queue;
 
   case BuiltinType::OCLReserveID:
     return OCLTK_ReserveID;
 
   case BuiltinType::OCLSampler:
     return OCLTK_Sampler;
 
   default:
     return OCLTK_Default;
   }
 }
 
 LangAS ASTContext::getOpenCLTypeAddrSpace(const Type *T) const {
   return Target->getOpenCLTypeAddrSpace(getOpenCLTypeKind(T));
 }
 
 /// BlockRequiresCopying - Returns true if byref variable "D" of type "Ty"
 /// requires copy/dispose. Note that this must match the logic
 /// in buildByrefHelpers.
 bool ASTContext::BlockRequiresCopying(QualType Ty,
                                       const VarDecl *D) {
   if (const CXXRecordDecl *record = Ty->getAsCXXRecordDecl()) {
     const Expr *copyExpr = getBlockVarCopyInit(D).getCopyExpr();
     if (!copyExpr && record->hasTrivialDestructor()) return false;
 
     return true;
   }
 
   // The block needs copy/destroy helpers if Ty is non-trivial to destructively
   // move or destroy.
   if (Ty.isNonTrivialToPrimitiveDestructiveMove() || Ty.isDestructedType())
     return true;
 
   if (!Ty->isObjCRetainableType()) return false;
 
   Qualifiers qs = Ty.getQualifiers();
 
   // If we have lifetime, that dominates.
   if (Qualifiers::ObjCLifetime lifetime = qs.getObjCLifetime()) {
     switch (lifetime) {
       case Qualifiers::OCL_None: llvm_unreachable("impossible");
 
       // These are just bits as far as the runtime is concerned.
       case Qualifiers::OCL_ExplicitNone:
       case Qualifiers::OCL_Autoreleasing:
         return false;
 
       // These cases should have been taken care of when checking the type's
       // non-triviality.
       case Qualifiers::OCL_Weak:
       case Qualifiers::OCL_Strong:
         llvm_unreachable("impossible");
     }
     llvm_unreachable("fell out of lifetime switch!");
   }
   return (Ty->isBlockPointerType() || isObjCNSObjectType(Ty) ||
           Ty->isObjCObjectPointerType());
 }
 
 bool ASTContext::getByrefLifetime(QualType Ty,
                               Qualifiers::ObjCLifetime &LifeTime,
                               bool &HasByrefExtendedLayout) const {
   if (!getLangOpts().ObjC ||
       getLangOpts().getGC() != LangOptions::NonGC)
     return false;
 
   HasByrefExtendedLayout = false;
   if (Ty->isRecordType()) {
     HasByrefExtendedLayout = true;
     LifeTime = Qualifiers::OCL_None;
   } else if ((LifeTime = Ty.getObjCLifetime())) {
     // Honor the ARC qualifiers.
   } else if (Ty->isObjCObjectPointerType() || Ty->isBlockPointerType()) {
     // The MRR rule.
     LifeTime = Qualifiers::OCL_ExplicitNone;
   } else {
     LifeTime = Qualifiers::OCL_None;
   }
   return true;
 }
 
 CanQualType ASTContext::getNSUIntegerType() const {
   assert(Target && "Expected target to be initialized");
   const llvm::Triple &T = Target->getTriple();
   // Windows is LLP64 rather than LP64
   if (T.isOSWindows() && T.isArch64Bit())
     return UnsignedLongLongTy;
   return UnsignedLongTy;
 }
 
 CanQualType ASTContext::getNSIntegerType() const {
   assert(Target && "Expected target to be initialized");
   const llvm::Triple &T = Target->getTriple();
   // Windows is LLP64 rather than LP64
   if (T.isOSWindows() && T.isArch64Bit())
     return LongLongTy;
   return LongTy;
 }
 
 TypedefDecl *ASTContext::getObjCInstanceTypeDecl() {
   if (!ObjCInstanceTypeDecl)
     ObjCInstanceTypeDecl =
         buildImplicitTypedef(getObjCIdType(), "instancetype");
   return ObjCInstanceTypeDecl;
 }
 
 // This returns true if a type has been typedefed to BOOL:
 // typedef <type> BOOL;
 static bool isTypeTypedefedAsBOOL(QualType T) {
   if (const auto *TT = dyn_cast<TypedefType>(T))
     if (IdentifierInfo *II = TT->getDecl()->getIdentifier())
       return II->isStr("BOOL");
 
   return false;
 }
 
 /// getObjCEncodingTypeSize returns size of type for objective-c encoding
 /// purpose.
 CharUnits ASTContext::getObjCEncodingTypeSize(QualType type) const {
   if (!type->isIncompleteArrayType() && type->isIncompleteType())
     return CharUnits::Zero();
 
   CharUnits sz = getTypeSizeInChars(type);
 
   // Make all integer and enum types at least as large as an int
   if (sz.isPositive() && type->isIntegralOrEnumerationType())
     sz = std::max(sz, getTypeSizeInChars(IntTy));
   // Treat arrays as pointers, since that's how they're passed in.
   else if (type->isArrayType())
     sz = getTypeSizeInChars(VoidPtrTy);
   return sz;
 }
 
 bool ASTContext::isMSStaticDataMemberInlineDefinition(const VarDecl *VD) const {
   return getTargetInfo().getCXXABI().isMicrosoft() &&
          VD->isStaticDataMember() &&
          VD->getType()->isIntegralOrEnumerationType() &&
          !VD->getFirstDecl()->isOutOfLine() && VD->getFirstDecl()->hasInit();
 }
 
 ASTContext::InlineVariableDefinitionKind
 ASTContext::getInlineVariableDefinitionKind(const VarDecl *VD) const {
   if (!VD->isInline())
     return InlineVariableDefinitionKind::None;
 
   // In almost all cases, it's a weak definition.
   auto *First = VD->getFirstDecl();
   if (First->isInlineSpecified() || !First->isStaticDataMember())
     return InlineVariableDefinitionKind::Weak;
 
   // If there's a file-context declaration in this translation unit, it's a
   // non-discardable definition.
   for (auto *D : VD->redecls())
     if (D->getLexicalDeclContext()->isFileContext() &&
         !D->isInlineSpecified() && (D->isConstexpr() || First->isConstexpr()))
       return InlineVariableDefinitionKind::Strong;
 
   // If we've not seen one yet, we don't know.
   return InlineVariableDefinitionKind::WeakUnknown;
 }
 
 static std::string charUnitsToString(const CharUnits &CU) {
   return llvm::itostr(CU.getQuantity());
 }
 
 /// getObjCEncodingForBlock - Return the encoded type for this block
 /// declaration.
 std::string ASTContext::getObjCEncodingForBlock(const BlockExpr *Expr) const {
   std::string S;
 
   const BlockDecl *Decl = Expr->getBlockDecl();
   QualType BlockTy =
       Expr->getType()->castAs<BlockPointerType>()->getPointeeType();
   QualType BlockReturnTy = BlockTy->castAs<FunctionType>()->getReturnType();
   // Encode result type.
   if (getLangOpts().EncodeExtendedBlockSig)
     getObjCEncodingForMethodParameter(Decl::OBJC_TQ_None, BlockReturnTy, S,
                                       true /*Extended*/);
   else
     getObjCEncodingForType(BlockReturnTy, S);
   // Compute size of all parameters.
   // Start with computing size of a pointer in number of bytes.
   // FIXME: There might(should) be a better way of doing this computation!
   CharUnits PtrSize = getTypeSizeInChars(VoidPtrTy);
   CharUnits ParmOffset = PtrSize;
   for (auto PI : Decl->parameters()) {
     QualType PType = PI->getType();
     CharUnits sz = getObjCEncodingTypeSize(PType);
     if (sz.isZero())
       continue;
     assert(sz.isPositive() && "BlockExpr - Incomplete param type");
     ParmOffset += sz;
   }
   // Size of the argument frame
   S += charUnitsToString(ParmOffset);
   // Block pointer and offset.
   S += "@?0";
 
   // Argument types.
   ParmOffset = PtrSize;
   for (auto PVDecl : Decl->parameters()) {
     QualType PType = PVDecl->getOriginalType();
     if (const auto *AT =
             dyn_cast<ArrayType>(PType->getCanonicalTypeInternal())) {
       // Use array's original type only if it has known number of
       // elements.
       if (!isa<ConstantArrayType>(AT))
         PType = PVDecl->getType();
     } else if (PType->isFunctionType())
       PType = PVDecl->getType();
     if (getLangOpts().EncodeExtendedBlockSig)
       getObjCEncodingForMethodParameter(Decl::OBJC_TQ_None, PType,
                                       S, true /*Extended*/);
     else
       getObjCEncodingForType(PType, S);
     S += charUnitsToString(ParmOffset);
     ParmOffset += getObjCEncodingTypeSize(PType);
   }
 
   return S;
 }
 
 std::string
 ASTContext::getObjCEncodingForFunctionDecl(const FunctionDecl *Decl) const {
   std::string S;
   // Encode result type.
   getObjCEncodingForType(Decl->getReturnType(), S);
   CharUnits ParmOffset;
   // Compute size of all parameters.
   for (auto PI : Decl->parameters()) {
     QualType PType = PI->getType();
     CharUnits sz = getObjCEncodingTypeSize(PType);
     if (sz.isZero())
       continue;
 
     assert(sz.isPositive() &&
            "getObjCEncodingForFunctionDecl - Incomplete param type");
     ParmOffset += sz;
   }
   S += charUnitsToString(ParmOffset);
   ParmOffset = CharUnits::Zero();
 
   // Argument types.
   for (auto PVDecl : Decl->parameters()) {
     QualType PType = PVDecl->getOriginalType();
     if (const auto *AT =
             dyn_cast<ArrayType>(PType->getCanonicalTypeInternal())) {
       // Use array's original type only if it has known number of
       // elements.
       if (!isa<ConstantArrayType>(AT))
         PType = PVDecl->getType();
     } else if (PType->isFunctionType())
       PType = PVDecl->getType();
     getObjCEncodingForType(PType, S);
     S += charUnitsToString(ParmOffset);
     ParmOffset += getObjCEncodingTypeSize(PType);
   }
 
   return S;
 }
 
 /// getObjCEncodingForMethodParameter - Return the encoded type for a single
 /// method parameter or return type. If Extended, include class names and
 /// block object types.
 void ASTContext::getObjCEncodingForMethodParameter(Decl::ObjCDeclQualifier QT,
                                                    QualType T, std::string& S,
                                                    bool Extended) const {
   // Encode type qualifier, 'in', 'inout', etc. for the parameter.
   getObjCEncodingForTypeQualifier(QT, S);
   // Encode parameter type.
   ObjCEncOptions Options = ObjCEncOptions()
                                .setExpandPointedToStructures()
                                .setExpandStructures()
                                .setIsOutermostType();
   if (Extended)
     Options.setEncodeBlockParameters().setEncodeClassNames();
   getObjCEncodingForTypeImpl(T, S, Options, /*Field=*/nullptr);
 }
 
 /// getObjCEncodingForMethodDecl - Return the encoded type for this method
 /// declaration.
 std::string ASTContext::getObjCEncodingForMethodDecl(const ObjCMethodDecl *Decl,
                                                      bool Extended) const {
   // FIXME: This is not very efficient.
   // Encode return type.
   std::string S;
   getObjCEncodingForMethodParameter(Decl->getObjCDeclQualifier(),
                                     Decl->getReturnType(), S, Extended);
   // Compute size of all parameters.
   // Start with computing size of a pointer in number of bytes.
   // FIXME: There might(should) be a better way of doing this computation!
   CharUnits PtrSize = getTypeSizeInChars(VoidPtrTy);
   // The first two arguments (self and _cmd) are pointers; account for
   // their size.
   CharUnits ParmOffset = 2 * PtrSize;
   for (ObjCMethodDecl::param_const_iterator PI = Decl->param_begin(),
        E = Decl->sel_param_end(); PI != E; ++PI) {
     QualType PType = (*PI)->getType();
     CharUnits sz = getObjCEncodingTypeSize(PType);
     if (sz.isZero())
       continue;
 
     assert(sz.isPositive() &&
            "getObjCEncodingForMethodDecl - Incomplete param type");
     ParmOffset += sz;
   }
   S += charUnitsToString(ParmOffset);
   S += "@0:";
   S += charUnitsToString(PtrSize);
 
   // Argument types.
   ParmOffset = 2 * PtrSize;
   for (ObjCMethodDecl::param_const_iterator PI = Decl->param_begin(),
        E = Decl->sel_param_end(); PI != E; ++PI) {
     const ParmVarDecl *PVDecl = *PI;
     QualType PType = PVDecl->getOriginalType();
     if (const auto *AT =
             dyn_cast<ArrayType>(PType->getCanonicalTypeInternal())) {
       // Use array's original type only if it has known number of
       // elements.
       if (!isa<ConstantArrayType>(AT))
         PType = PVDecl->getType();
     } else if (PType->isFunctionType())
       PType = PVDecl->getType();
     getObjCEncodingForMethodParameter(PVDecl->getObjCDeclQualifier(),
                                       PType, S, Extended);
     S += charUnitsToString(ParmOffset);
     ParmOffset += getObjCEncodingTypeSize(PType);
   }
 
   return S;
 }
 
 ObjCPropertyImplDecl *
 ASTContext::getObjCPropertyImplDeclForPropertyDecl(
                                       const ObjCPropertyDecl *PD,
                                       const Decl *Container) const {
   if (!Container)
     return nullptr;
   if (const auto *CID = dyn_cast<ObjCCategoryImplDecl>(Container)) {
     for (auto *PID : CID->property_impls())
       if (PID->getPropertyDecl() == PD)
         return PID;
   } else {
     const auto *OID = cast<ObjCImplementationDecl>(Container);
     for (auto *PID : OID->property_impls())
       if (PID->getPropertyDecl() == PD)
         return PID;
   }
   return nullptr;
 }
 
 /// getObjCEncodingForPropertyDecl - Return the encoded type for this
 /// property declaration. If non-NULL, Container must be either an
 /// ObjCCategoryImplDecl or ObjCImplementationDecl; it should only be
 /// NULL when getting encodings for protocol properties.
 /// Property attributes are stored as a comma-delimited C string. The simple
 /// attributes readonly and bycopy are encoded as single characters. The
 /// parametrized attributes, getter=name, setter=name, and ivar=name, are
 /// encoded as single characters, followed by an identifier. Property types
 /// are also encoded as a parametrized attribute. The characters used to encode
 /// these attributes are defined by the following enumeration:
 /// @code
 /// enum PropertyAttributes {
 /// kPropertyReadOnly = 'R',   // property is read-only.
 /// kPropertyBycopy = 'C',     // property is a copy of the value last assigned
 /// kPropertyByref = '&',  // property is a reference to the value last assigned
 /// kPropertyDynamic = 'D',    // property is dynamic
 /// kPropertyGetter = 'G',     // followed by getter selector name
 /// kPropertySetter = 'S',     // followed by setter selector name
 /// kPropertyInstanceVariable = 'V'  // followed by instance variable  name
 /// kPropertyType = 'T'              // followed by old-style type encoding.
 /// kPropertyWeak = 'W'              // 'weak' property
 /// kPropertyStrong = 'P'            // property GC'able
 /// kPropertyNonAtomic = 'N'         // property non-atomic
 /// };
 /// @endcode
 std::string
 ASTContext::getObjCEncodingForPropertyDecl(const ObjCPropertyDecl *PD,
                                            const Decl *Container) const {
   // Collect information from the property implementation decl(s).
   bool Dynamic = false;
   ObjCPropertyImplDecl *SynthesizePID = nullptr;
 
   if (ObjCPropertyImplDecl *PropertyImpDecl =
       getObjCPropertyImplDeclForPropertyDecl(PD, Container)) {
     if (PropertyImpDecl->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
       Dynamic = true;
     else
       SynthesizePID = PropertyImpDecl;
   }
 
   // FIXME: This is not very efficient.
   std::string S = "T";
 
   // Encode result type.
   // GCC has some special rules regarding encoding of properties which
   // closely resembles encoding of ivars.
   getObjCEncodingForPropertyType(PD->getType(), S);
 
   if (PD->isReadOnly()) {
     S += ",R";
     if (PD->getPropertyAttributes() & ObjCPropertyAttribute::kind_copy)
       S += ",C";
     if (PD->getPropertyAttributes() & ObjCPropertyAttribute::kind_retain)
       S += ",&";
     if (PD->getPropertyAttributes() & ObjCPropertyAttribute::kind_weak)
       S += ",W";
   } else {
     switch (PD->getSetterKind()) {
     case ObjCPropertyDecl::Assign: break;
     case ObjCPropertyDecl::Copy:   S += ",C"; break;
     case ObjCPropertyDecl::Retain: S += ",&"; break;
     case ObjCPropertyDecl::Weak:   S += ",W"; break;
     }
   }
 
   // It really isn't clear at all what this means, since properties
   // are "dynamic by default".
   if (Dynamic)
     S += ",D";
 
   if (PD->getPropertyAttributes() & ObjCPropertyAttribute::kind_nonatomic)
     S += ",N";
 
   if (PD->getPropertyAttributes() & ObjCPropertyAttribute::kind_getter) {
     S += ",G";
     S += PD->getGetterName().getAsString();
   }
 
   if (PD->getPropertyAttributes() & ObjCPropertyAttribute::kind_setter) {
     S += ",S";
     S += PD->getSetterName().getAsString();
   }
 
   if (SynthesizePID) {
     const ObjCIvarDecl *OID = SynthesizePID->getPropertyIvarDecl();
     S += ",V";
     S += OID->getNameAsString();
   }
 
   // FIXME: OBJCGC: weak & strong
   return S;
 }
 
 /// getLegacyIntegralTypeEncoding -
 /// Another legacy compatibility encoding: 32-bit longs are encoded as
 /// 'l' or 'L' , but not always.  For typedefs, we need to use
 /// 'i' or 'I' instead if encoding a struct field, or a pointer!
 void ASTContext::getLegacyIntegralTypeEncoding (QualType &PointeeTy) const {
   if (isa<TypedefType>(PointeeTy.getTypePtr())) {
     if (const auto *BT = PointeeTy->getAs<BuiltinType>()) {
       if (BT->getKind() == BuiltinType::ULong && getIntWidth(PointeeTy) == 32)
         PointeeTy = UnsignedIntTy;
       else
         if (BT->getKind() == BuiltinType::Long && getIntWidth(PointeeTy) == 32)
           PointeeTy = IntTy;
     }
   }
 }
 
 void ASTContext::getObjCEncodingForType(QualType T, std::string& S,
                                         const FieldDecl *Field,
                                         QualType *NotEncodedT) const {
   // We follow the behavior of gcc, expanding structures which are
   // directly pointed to, and expanding embedded structures. Note that
   // these rules are sufficient to prevent recursive encoding of the
   // same type.
   getObjCEncodingForTypeImpl(T, S,
                              ObjCEncOptions()
                                  .setExpandPointedToStructures()
                                  .setExpandStructures()
                                  .setIsOutermostType(),
                              Field, NotEncodedT);
 }
 
 void ASTContext::getObjCEncodingForPropertyType(QualType T,
                                                 std::string& S) const {
   // Encode result type.
   // GCC has some special rules regarding encoding of properties which
   // closely resembles encoding of ivars.
   getObjCEncodingForTypeImpl(T, S,
                              ObjCEncOptions()
                                  .setExpandPointedToStructures()
                                  .setExpandStructures()
                                  .setIsOutermostType()
                                  .setEncodingProperty(),
                              /*Field=*/nullptr);
 }
 
 static char getObjCEncodingForPrimitiveType(const ASTContext *C,
                                             const BuiltinType *BT) {
     BuiltinType::Kind kind = BT->getKind();
     switch (kind) {
     case BuiltinType::Void:       return 'v';
     case BuiltinType::Bool:       return 'B';
     case BuiltinType::Char8:
     case BuiltinType::Char_U:
     case BuiltinType::UChar:      return 'C';
     case BuiltinType::Char16:
     case BuiltinType::UShort:     return 'S';
     case BuiltinType::Char32:
     case BuiltinType::UInt:       return 'I';
     case BuiltinType::ULong:
         return C->getTargetInfo().getLongWidth() == 32 ? 'L' : 'Q';
     case BuiltinType::UInt128:    return 'T';
     case BuiltinType::ULongLong:  return 'Q';
     case BuiltinType::Char_S:
     case BuiltinType::SChar:      return 'c';
     case BuiltinType::Short:      return 's';
     case BuiltinType::WChar_S:
     case BuiltinType::WChar_U:
     case BuiltinType::Int:        return 'i';
     case BuiltinType::Long:
       return C->getTargetInfo().getLongWidth() == 32 ? 'l' : 'q';
     case BuiltinType::LongLong:   return 'q';
     case BuiltinType::Int128:     return 't';
     case BuiltinType::Float:      return 'f';
     case BuiltinType::Double:     return 'd';
     case BuiltinType::LongDouble: return 'D';
     case BuiltinType::NullPtr:    return '*'; // like char*
 
     case BuiltinType::BFloat16:
     case BuiltinType::Float16:
     case BuiltinType::Float128:
     case BuiltinType::Ibm128:
     case BuiltinType::Half:
     case BuiltinType::ShortAccum:
     case BuiltinType::Accum:
     case BuiltinType::LongAccum:
     case BuiltinType::UShortAccum:
     case BuiltinType::UAccum:
     case BuiltinType::ULongAccum:
     case BuiltinType::ShortFract:
     case BuiltinType::Fract:
     case BuiltinType::LongFract:
     case BuiltinType::UShortFract:
     case BuiltinType::UFract:
     case BuiltinType::ULongFract:
     case BuiltinType::SatShortAccum:
     case BuiltinType::SatAccum:
     case BuiltinType::SatLongAccum:
     case BuiltinType::SatUShortAccum:
     case BuiltinType::SatUAccum:
     case BuiltinType::SatULongAccum:
     case BuiltinType::SatShortFract:
     case BuiltinType::SatFract:
     case BuiltinType::SatLongFract:
     case BuiltinType::SatUShortFract:
     case BuiltinType::SatUFract:
     case BuiltinType::SatULongFract:
       // FIXME: potentially need @encodes for these!
       return ' ';
 
 #define SVE_TYPE(Name, Id, SingletonId) \
     case BuiltinType::Id:
 #include "clang/Basic/AArch64SVEACLETypes.def"
 #define RVV_TYPE(Name, Id, SingletonId) case BuiltinType::Id:
 #include "clang/Basic/RISCVVTypes.def"
       {
         DiagnosticsEngine &Diags = C->getDiagnostics();
         unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
                                                 "cannot yet @encode type %0");
         Diags.Report(DiagID) << BT->getName(C->getPrintingPolicy());
         return ' ';
       }
 
     case BuiltinType::ObjCId:
     case BuiltinType::ObjCClass:
     case BuiltinType::ObjCSel:
       llvm_unreachable("@encoding ObjC primitive type");
 
     // OpenCL and placeholder types don't need @encodings.
 #define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
     case BuiltinType::Id:
 #include "clang/Basic/OpenCLImageTypes.def"
 #define EXT_OPAQUE_TYPE(ExtType, Id, Ext) \
     case BuiltinType::Id:
 #include "clang/Basic/OpenCLExtensionTypes.def"
     case BuiltinType::OCLEvent:
     case BuiltinType::OCLClkEvent:
     case BuiltinType::OCLQueue:
     case BuiltinType::OCLReserveID:
     case BuiltinType::OCLSampler:
     case BuiltinType::Dependent:
 #define PPC_VECTOR_TYPE(Name, Id, Size) \
     case BuiltinType::Id:
 #include "clang/Basic/PPCTypes.def"
 #define BUILTIN_TYPE(KIND, ID)
 #define PLACEHOLDER_TYPE(KIND, ID) \
     case BuiltinType::KIND:
 #include "clang/AST/BuiltinTypes.def"
       llvm_unreachable("invalid builtin type for @encode");
     }
     llvm_unreachable("invalid BuiltinType::Kind value");
 }
 
 static char ObjCEncodingForEnumType(const ASTContext *C, const EnumType *ET) {
   EnumDecl *Enum = ET->getDecl();
 
   // The encoding of an non-fixed enum type is always 'i', regardless of size.
   if (!Enum->isFixed())
     return 'i';
 
   // The encoding of a fixed enum type matches its fixed underlying type.
   const auto *BT = Enum->getIntegerType()->castAs<BuiltinType>();
   return getObjCEncodingForPrimitiveType(C, BT);
 }
 
 static void EncodeBitField(const ASTContext *Ctx, std::string& S,
                            QualType T, const FieldDecl *FD) {
   assert(FD->isBitField() && "not a bitfield - getObjCEncodingForTypeImpl");
   S += 'b';
   // The NeXT runtime encodes bit fields as b followed by the number of bits.
   // The GNU runtime requires more information; bitfields are encoded as b,
   // then the offset (in bits) of the first element, then the type of the
   // bitfield, then the size in bits.  For example, in this structure:
   //
   // struct
   // {
   //    int integer;
   //    int flags:2;
   // };
   // On a 32-bit system, the encoding for flags would be b2 for the NeXT
   // runtime, but b32i2 for the GNU runtime.  The reason for this extra
   // information is not especially sensible, but we're stuck with it for
   // compatibility with GCC, although providing it breaks anything that
   // actually uses runtime introspection and wants to work on both runtimes...
   if (Ctx->getLangOpts().ObjCRuntime.isGNUFamily()) {
     uint64_t Offset;
 
     if (const auto *IVD = dyn_cast<ObjCIvarDecl>(FD)) {
       Offset = Ctx->lookupFieldBitOffset(IVD->getContainingInterface(), nullptr,
                                          IVD);
     } else {
       const RecordDecl *RD = FD->getParent();
       const ASTRecordLayout &RL = Ctx->getASTRecordLayout(RD);
       Offset = RL.getFieldOffset(FD->getFieldIndex());
     }
 
     S += llvm::utostr(Offset);
 
     if (const auto *ET = T->getAs<EnumType>())
       S += ObjCEncodingForEnumType(Ctx, ET);
     else {
       const auto *BT = T->castAs<BuiltinType>();
       S += getObjCEncodingForPrimitiveType(Ctx, BT);
     }
   }
   S += llvm::utostr(FD->getBitWidthValue(*Ctx));
 }
 
 // Helper function for determining whether the encoded type string would include
 // a template specialization type.
 static bool hasTemplateSpecializationInEncodedString(const Type *T,
                                                      bool VisitBasesAndFields) {
   T = T->getBaseElementTypeUnsafe();
 
   if (auto *PT = T->getAs<PointerType>())
     return hasTemplateSpecializationInEncodedString(
         PT->getPointeeType().getTypePtr(), false);
 
   auto *CXXRD = T->getAsCXXRecordDecl();
 
   if (!CXXRD)
     return false;
 
   if (isa<ClassTemplateSpecializationDecl>(CXXRD))
     return true;
 
   if (!CXXRD->hasDefinition() || !VisitBasesAndFields)
     return false;
 
   for (auto B : CXXRD->bases())
     if (hasTemplateSpecializationInEncodedString(B.getType().getTypePtr(),
                                                  true))
       return true;
 
   for (auto *FD : CXXRD->fields())
     if (hasTemplateSpecializationInEncodedString(FD->getType().getTypePtr(),
                                                  true))
       return true;
 
   return false;
 }
 
 // FIXME: Use SmallString for accumulating string.
 void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string &S,
                                             const ObjCEncOptions Options,
                                             const FieldDecl *FD,
                                             QualType *NotEncodedT) const {
   CanQualType CT = getCanonicalType(T);
   switch (CT->getTypeClass()) {
   case Type::Builtin:
   case Type::Enum:
     if (FD && FD->isBitField())
       return EncodeBitField(this, S, T, FD);
     if (const auto *BT = dyn_cast<BuiltinType>(CT))
       S += getObjCEncodingForPrimitiveType(this, BT);
     else
       S += ObjCEncodingForEnumType(this, cast<EnumType>(CT));
     return;
 
   case Type::Complex:
     S += 'j';
     getObjCEncodingForTypeImpl(T->castAs<ComplexType>()->getElementType(), S,
                                ObjCEncOptions(),
                                /*Field=*/nullptr);
     return;
 
   case Type::Atomic:
     S += 'A';
     getObjCEncodingForTypeImpl(T->castAs<AtomicType>()->getValueType(), S,
                                ObjCEncOptions(),
                                /*Field=*/nullptr);
     return;
 
   // encoding for pointer or reference types.
   case Type::Pointer:
   case Type::LValueReference:
   case Type::RValueReference: {
     QualType PointeeTy;
     if (isa<PointerType>(CT)) {
       const auto *PT = T->castAs<PointerType>();
       if (PT->isObjCSelType()) {
         S += ':';
         return;
       }
       PointeeTy = PT->getPointeeType();
     } else {
       PointeeTy = T->castAs<ReferenceType>()->getPointeeType();
     }
 
     bool isReadOnly = false;
     // For historical/compatibility reasons, the read-only qualifier of the
     // pointee gets emitted _before_ the '^'.  The read-only qualifier of
     // the pointer itself gets ignored, _unless_ we are looking at a typedef!
     // Also, do not emit the 'r' for anything but the outermost type!
     if (isa<TypedefType>(T.getTypePtr())) {
       if (Options.IsOutermostType() && T.isConstQualified()) {
         isReadOnly = true;
         S += 'r';
       }
     } else if (Options.IsOutermostType()) {
       QualType P = PointeeTy;
       while (auto PT = P->getAs<PointerType>())
         P = PT->getPointeeType();
       if (P.isConstQualified()) {
         isReadOnly = true;
         S += 'r';
       }
     }
     if (isReadOnly) {
       // Another legacy compatibility encoding. Some ObjC qualifier and type
       // combinations need to be rearranged.
       // Rewrite "in const" from "nr" to "rn"
       if (StringRef(S).endswith("nr"))
         S.replace(S.end()-2, S.end(), "rn");
     }
 
     if (PointeeTy->isCharType()) {
       // char pointer types should be encoded as '*' unless it is a
       // type that has been typedef'd to 'BOOL'.
       if (!isTypeTypedefedAsBOOL(PointeeTy)) {
         S += '*';
         return;
       }
     } else if (const auto *RTy = PointeeTy->getAs<RecordType>()) {
       // GCC binary compat: Need to convert "struct objc_class *" to "#".
       if (RTy->getDecl()->getIdentifier() == &Idents.get("objc_class")) {
         S += '#';
         return;
       }
       // GCC binary compat: Need to convert "struct objc_object *" to "@".
       if (RTy->getDecl()->getIdentifier() == &Idents.get("objc_object")) {
         S += '@';
         return;
       }
       // If the encoded string for the class includes template names, just emit
       // "^v" for pointers to the class.
       if (getLangOpts().CPlusPlus &&
           (!getLangOpts().EncodeCXXClassTemplateSpec &&
            hasTemplateSpecializationInEncodedString(
                RTy, Options.ExpandPointedToStructures()))) {
         S += "^v";
         return;
       }
       // fall through...
     }
     S += '^';
     getLegacyIntegralTypeEncoding(PointeeTy);
 
     ObjCEncOptions NewOptions;
     if (Options.ExpandPointedToStructures())
       NewOptions.setExpandStructures();
     getObjCEncodingForTypeImpl(PointeeTy, S, NewOptions,
                                /*Field=*/nullptr, NotEncodedT);
     return;
   }
 
   case Type::ConstantArray:
   case Type::IncompleteArray:
   case Type::VariableArray: {
     const auto *AT = cast<ArrayType>(CT);
 
     if (isa<IncompleteArrayType>(AT) && !Options.IsStructField()) {
       // Incomplete arrays are encoded as a pointer to the array element.
       S += '^';
 
       getObjCEncodingForTypeImpl(
           AT->getElementType(), S,
           Options.keepingOnly(ObjCEncOptions().setExpandStructures()), FD);
     } else {
       S += '[';
 
       if (const auto *CAT = dyn_cast<ConstantArrayType>(AT))
         S += llvm::utostr(CAT->getSize().getZExtValue());
       else {
         //Variable length arrays are encoded as a regular array with 0 elements.
         assert((isa<VariableArrayType>(AT) || isa<IncompleteArrayType>(AT)) &&
                "Unknown array type!");
         S += '0';
       }
 
       getObjCEncodingForTypeImpl(
           AT->getElementType(), S,
           Options.keepingOnly(ObjCEncOptions().setExpandStructures()), FD,
           NotEncodedT);
       S += ']';
     }
     return;
   }
 
   case Type::FunctionNoProto:
   case Type::FunctionProto:
     S += '?';
     return;
 
   case Type::Record: {
     RecordDecl *RDecl = cast<RecordType>(CT)->getDecl();
     S += RDecl->isUnion() ? '(' : '{';
     // Anonymous structures print as '?'
     if (const IdentifierInfo *II = RDecl->getIdentifier()) {
       S += II->getName();
       if (const auto *Spec = dyn_cast<ClassTemplateSpecializationDecl>(RDecl)) {
         const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
         llvm::raw_string_ostream OS(S);
         printTemplateArgumentList(OS, TemplateArgs.asArray(),
                                   getPrintingPolicy());
       }
     } else {
       S += '?';
     }
     if (Options.ExpandStructures()) {
       S += '=';
       if (!RDecl->isUnion()) {
         getObjCEncodingForStructureImpl(RDecl, S, FD, true, NotEncodedT);
       } else {
         for (const auto *Field : RDecl->fields()) {
           if (FD) {
             S += '"';
             S += Field->getNameAsString();
             S += '"';
           }
 
           // Special case bit-fields.
           if (Field->isBitField()) {
             getObjCEncodingForTypeImpl(Field->getType(), S,
                                        ObjCEncOptions().setExpandStructures(),
                                        Field);
           } else {
             QualType qt = Field->getType();
             getLegacyIntegralTypeEncoding(qt);
             getObjCEncodingForTypeImpl(
                 qt, S,
                 ObjCEncOptions().setExpandStructures().setIsStructField(), FD,
                 NotEncodedT);
           }
         }
       }
     }
     S += RDecl->isUnion() ? ')' : '}';
     return;
   }
 
   case Type::BlockPointer: {
     const auto *BT = T->castAs<BlockPointerType>();
     S += "@?"; // Unlike a pointer-to-function, which is "^?".
     if (Options.EncodeBlockParameters()) {
       const auto *FT = BT->getPointeeType()->castAs<FunctionType>();
 
       S += '<';
       // Block return type
       getObjCEncodingForTypeImpl(FT->getReturnType(), S,
                                  Options.forComponentType(), FD, NotEncodedT);
       // Block self
       S += "@?";
       // Block parameters
       if (const auto *FPT = dyn_cast<FunctionProtoType>(FT)) {
         for (const auto &I : FPT->param_types())
           getObjCEncodingForTypeImpl(I, S, Options.forComponentType(), FD,
                                      NotEncodedT);
       }
       S += '>';
     }
     return;
   }
 
   case Type::ObjCObject: {
     // hack to match legacy encoding of *id and *Class
     QualType Ty = getObjCObjectPointerType(CT);
     if (Ty->isObjCIdType()) {
       S += "{objc_object=}";
       return;
     }
     else if (Ty->isObjCClassType()) {
       S += "{objc_class=}";
       return;
     }
     // TODO: Double check to make sure this intentionally falls through.
     LLVM_FALLTHROUGH;
   }
 
   case Type::ObjCInterface: {
     // Ignore protocol qualifiers when mangling at this level.
     // @encode(class_name)
     ObjCInterfaceDecl *OI = T->castAs<ObjCObjectType>()->getInterface();
     S += '{';
     S += OI->getObjCRuntimeNameAsString();
     if (Options.ExpandStructures()) {
       S += '=';
       SmallVector<const ObjCIvarDecl*, 32> Ivars;
       DeepCollectObjCIvars(OI, true, Ivars);
       for (unsigned i = 0, e = Ivars.size(); i != e; ++i) {
         const FieldDecl *Field = Ivars[i];
         if (Field->isBitField())
           getObjCEncodingForTypeImpl(Field->getType(), S,
                                      ObjCEncOptions().setExpandStructures(),
                                      Field);
         else
           getObjCEncodingForTypeImpl(Field->getType(), S,
                                      ObjCEncOptions().setExpandStructures(), FD,
                                      NotEncodedT);
       }
     }
     S += '}';
     return;
   }
 
   case Type::ObjCObjectPointer: {
     const auto *OPT = T->castAs<ObjCObjectPointerType>();
     if (OPT->isObjCIdType()) {
       S += '@';
       return;
     }
 
     if (OPT->isObjCClassType() || OPT->isObjCQualifiedClassType()) {
       // FIXME: Consider if we need to output qualifiers for 'Class<p>'.
       // Since this is a binary compatibility issue, need to consult with
       // runtime folks. Fortunately, this is a *very* obscure construct.
       S += '#';
       return;
     }
 
     if (OPT->isObjCQualifiedIdType()) {
       getObjCEncodingForTypeImpl(
           getObjCIdType(), S,
           Options.keepingOnly(ObjCEncOptions()
                                   .setExpandPointedToStructures()
                                   .setExpandStructures()),
           FD);
       if (FD || Options.EncodingProperty() || Options.EncodeClassNames()) {
         // Note that we do extended encoding of protocol qualifier list
         // Only when doing ivar or property encoding.
         S += '"';
         for (const auto *I : OPT->quals()) {
           S += '<';
           S += I->getObjCRuntimeNameAsString();
           S += '>';
         }
         S += '"';
       }
       return;
     }
 
     S += '@';
     if (OPT->getInterfaceDecl() &&
         (FD || Options.EncodingProperty() || Options.EncodeClassNames())) {
       S += '"';
       S += OPT->getInterfaceDecl()->getObjCRuntimeNameAsString();
       for (const auto *I : OPT->quals()) {
         S += '<';
         S += I->getObjCRuntimeNameAsString();
         S += '>';
       }
       S += '"';
     }
     return;
   }
 
   // gcc just blithely ignores member pointers.
   // FIXME: we should do better than that.  'M' is available.
   case Type::MemberPointer:
   // This matches gcc's encoding, even though technically it is insufficient.
   //FIXME. We should do a better job than gcc.
   case Type::Vector:
   case Type::ExtVector:
   // Until we have a coherent encoding of these three types, issue warning.
     if (NotEncodedT)
       *NotEncodedT = T;
     return;
 
   case Type::ConstantMatrix:
     if (NotEncodedT)
       *NotEncodedT = T;
     return;
 
   case Type::BitInt:
     if (NotEncodedT)
       *NotEncodedT = T;
     return;
 
   // We could see an undeduced auto type here during error recovery.
   // Just ignore it.
   case Type::Auto:
   case Type::DeducedTemplateSpecialization:
     return;
 
   case Type::Pipe:
 #define ABSTRACT_TYPE(KIND, BASE)
 #define TYPE(KIND, BASE)
 #define DEPENDENT_TYPE(KIND, BASE) \
   case Type::KIND:
 #define NON_CANONICAL_TYPE(KIND, BASE) \
   case Type::KIND:
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(KIND, BASE) \
   case Type::KIND:
 #include "clang/AST/TypeNodes.inc"
     llvm_unreachable("@encode for dependent type!");
   }
   llvm_unreachable("bad type kind!");
 }
 
 void ASTContext::getObjCEncodingForStructureImpl(RecordDecl *RDecl,
                                                  std::string &S,
                                                  const FieldDecl *FD,
                                                  bool includeVBases,
                                                  QualType *NotEncodedT) const {
   assert(RDecl && "Expected non-null RecordDecl");
   assert(!RDecl->isUnion() && "Should not be called for unions");
   if (!RDecl->getDefinition() || RDecl->getDefinition()->isInvalidDecl())
     return;
 
   const auto *CXXRec = dyn_cast<CXXRecordDecl>(RDecl);
   std::multimap<uint64_t, NamedDecl *> FieldOrBaseOffsets;
   const ASTRecordLayout &layout = getASTRecordLayout(RDecl);
 
   if (CXXRec) {
     for (const auto &BI : CXXRec->bases()) {
       if (!BI.isVirtual()) {
         CXXRecordDecl *base = BI.getType()->getAsCXXRecordDecl();
         if (base->isEmpty())
           continue;
         uint64_t offs = toBits(layout.getBaseClassOffset(base));
         FieldOrBaseOffsets.insert(FieldOrBaseOffsets.upper_bound(offs),
                                   std::make_pair(offs, base));
       }
     }
   }
 
   unsigned i = 0;
   for (FieldDecl *Field : RDecl->fields()) {
     if (!Field->isZeroLengthBitField(*this) && Field->isZeroSize(*this))
       continue;
     uint64_t offs = layout.getFieldOffset(i);
     FieldOrBaseOffsets.insert(FieldOrBaseOffsets.upper_bound(offs),
                               std::make_pair(offs, Field));
     ++i;
   }
 
   if (CXXRec && includeVBases) {
     for (const auto &BI : CXXRec->vbases()) {
       CXXRecordDecl *base = BI.getType()->getAsCXXRecordDecl();
       if (base->isEmpty())
         continue;
       uint64_t offs = toBits(layout.getVBaseClassOffset(base));
       if (offs >= uint64_t(toBits(layout.getNonVirtualSize())) &&
           FieldOrBaseOffsets.find(offs) == FieldOrBaseOffsets.end())
         FieldOrBaseOffsets.insert(FieldOrBaseOffsets.end(),
                                   std::make_pair(offs, base));
     }
   }
 
   CharUnits size;
   if (CXXRec) {
     size = includeVBases ? layout.getSize() : layout.getNonVirtualSize();
   } else {
     size = layout.getSize();
   }
 
 #ifndef NDEBUG
   uint64_t CurOffs = 0;
 #endif
   std::multimap<uint64_t, NamedDecl *>::iterator
     CurLayObj = FieldOrBaseOffsets.begin();
 
   if (CXXRec && CXXRec->isDynamicClass() &&
       (CurLayObj == FieldOrBaseOffsets.end() || CurLayObj->first != 0)) {
     if (FD) {
       S += "\"_vptr$";
       std::string recname = CXXRec->getNameAsString();
       if (recname.empty()) recname = "?";
       S += recname;
       S += '"';
     }
     S += "^^?";
 #ifndef NDEBUG
     CurOffs += getTypeSize(VoidPtrTy);
 #endif
   }
 
   if (!RDecl->hasFlexibleArrayMember()) {
     // Mark the end of the structure.
     uint64_t offs = toBits(size);
     FieldOrBaseOffsets.insert(FieldOrBaseOffsets.upper_bound(offs),
                               std::make_pair(offs, nullptr));
   }
 
   for (; CurLayObj != FieldOrBaseOffsets.end(); ++CurLayObj) {
 #ifndef NDEBUG
     assert(CurOffs <= CurLayObj->first);
     if (CurOffs < CurLayObj->first) {
       uint64_t padding = CurLayObj->first - CurOffs;
       // FIXME: There doesn't seem to be a way to indicate in the encoding that
       // packing/alignment of members is different that normal, in which case
       // the encoding will be out-of-sync with the real layout.
       // If the runtime switches to just consider the size of types without
       // taking into account alignment, we could make padding explicit in the
       // encoding (e.g. using arrays of chars). The encoding strings would be
       // longer then though.
       CurOffs += padding;
     }
 #endif
 
     NamedDecl *dcl = CurLayObj->second;
     if (!dcl)
       break; // reached end of structure.
 
     if (auto *base = dyn_cast<CXXRecordDecl>(dcl)) {
       // We expand the bases without their virtual bases since those are going
       // in the initial structure. Note that this differs from gcc which
       // expands virtual bases each time one is encountered in the hierarchy,
       // making the encoding type bigger than it really is.
       getObjCEncodingForStructureImpl(base, S, FD, /*includeVBases*/false,
                                       NotEncodedT);
       assert(!base->isEmpty());
 #ifndef NDEBUG
       CurOffs += toBits(getASTRecordLayout(base).getNonVirtualSize());
 #endif
     } else {
       const auto *field = cast<FieldDecl>(dcl);
       if (FD) {
         S += '"';
         S += field->getNameAsString();
         S += '"';
       }
 
       if (field->isBitField()) {
         EncodeBitField(this, S, field->getType(), field);
 #ifndef NDEBUG
         CurOffs += field->getBitWidthValue(*this);
 #endif
       } else {
         QualType qt = field->getType();
         getLegacyIntegralTypeEncoding(qt);
         getObjCEncodingForTypeImpl(
             qt, S, ObjCEncOptions().setExpandStructures().setIsStructField(),
             FD, NotEncodedT);
 #ifndef NDEBUG
         CurOffs += getTypeSize(field->getType());
 #endif
       }
     }
   }
 }
 
 void ASTContext::getObjCEncodingForTypeQualifier(Decl::ObjCDeclQualifier QT,
                                                  std::string& S) const {
   if (QT & Decl::OBJC_TQ_In)
     S += 'n';
   if (QT & Decl::OBJC_TQ_Inout)
     S += 'N';
   if (QT & Decl::OBJC_TQ_Out)
     S += 'o';
   if (QT & Decl::OBJC_TQ_Bycopy)
     S += 'O';
   if (QT & Decl::OBJC_TQ_Byref)
     S += 'R';
   if (QT & Decl::OBJC_TQ_Oneway)
     S += 'V';
 }
 
 TypedefDecl *ASTContext::getObjCIdDecl() const {
   if (!ObjCIdDecl) {
     QualType T = getObjCObjectType(ObjCBuiltinIdTy, {}, {});
     T = getObjCObjectPointerType(T);
     ObjCIdDecl = buildImplicitTypedef(T, "id");
   }
   return ObjCIdDecl;
 }
 
 TypedefDecl *ASTContext::getObjCSelDecl() const {
   if (!ObjCSelDecl) {
     QualType T = getPointerType(ObjCBuiltinSelTy);
     ObjCSelDecl = buildImplicitTypedef(T, "SEL");
   }
   return ObjCSelDecl;
 }
 
 TypedefDecl *ASTContext::getObjCClassDecl() const {
   if (!ObjCClassDecl) {
     QualType T = getObjCObjectType(ObjCBuiltinClassTy, {}, {});
     T = getObjCObjectPointerType(T);
     ObjCClassDecl = buildImplicitTypedef(T, "Class");
   }
   return ObjCClassDecl;
 }
 
 ObjCInterfaceDecl *ASTContext::getObjCProtocolDecl() const {
   if (!ObjCProtocolClassDecl) {
     ObjCProtocolClassDecl
       = ObjCInterfaceDecl::Create(*this, getTranslationUnitDecl(),
                                   SourceLocation(),
                                   &Idents.get("Protocol"),
                                   /*typeParamList=*/nullptr,
                                   /*PrevDecl=*/nullptr,
                                   SourceLocation(), true);
   }
 
   return ObjCProtocolClassDecl;
 }
 
 //===----------------------------------------------------------------------===//
 // __builtin_va_list Construction Functions
 //===----------------------------------------------------------------------===//
 
 static TypedefDecl *CreateCharPtrNamedVaListDecl(const ASTContext *Context,
                                                  StringRef Name) {
   // typedef char* __builtin[_ms]_va_list;
   QualType T = Context->getPointerType(Context->CharTy);
   return Context->buildImplicitTypedef(T, Name);
 }
 
 static TypedefDecl *CreateMSVaListDecl(const ASTContext *Context) {
   return CreateCharPtrNamedVaListDecl(Context, "__builtin_ms_va_list");
 }
 
 static TypedefDecl *CreateCharPtrBuiltinVaListDecl(const ASTContext *Context) {
   return CreateCharPtrNamedVaListDecl(Context, "__builtin_va_list");
 }
 
 static TypedefDecl *CreateVoidPtrBuiltinVaListDecl(const ASTContext *Context) {
   // typedef void* __builtin_va_list;
   QualType T = Context->getPointerType(Context->VoidTy);
   return Context->buildImplicitTypedef(T, "__builtin_va_list");
 }
 
 static TypedefDecl *
 CreateAArch64ABIBuiltinVaListDecl(const ASTContext *Context) {
   // struct __va_list
   RecordDecl *VaListTagDecl = Context->buildImplicitRecord("__va_list");
   if (Context->getLangOpts().CPlusPlus) {
     // namespace std { struct __va_list {
-    NamespaceDecl *NS;
-    NS = NamespaceDecl::Create(const_cast<ASTContext &>(*Context),
-                               Context->getTranslationUnitDecl(),
-                               /*Inline*/ false, SourceLocation(),
-                               SourceLocation(), &Context->Idents.get("std"),
-                               /*PrevDecl*/ nullptr);
+    auto *NS = NamespaceDecl::Create(
+        const_cast<ASTContext &>(*Context), Context->getTranslationUnitDecl(),
+        /*Inline*/ false, SourceLocation(), SourceLocation(),
+        &Context->Idents.get("std"),
+        /*PrevDecl*/ nullptr);
     NS->setImplicit();
     VaListTagDecl->setDeclContext(NS);
   }
 
   VaListTagDecl->startDefinition();
 
   const size_t NumFields = 5;
   QualType FieldTypes[NumFields];
   const char *FieldNames[NumFields];
 
   // void *__stack;
   FieldTypes[0] = Context->getPointerType(Context->VoidTy);
   FieldNames[0] = "__stack";
 
   // void *__gr_top;
   FieldTypes[1] = Context->getPointerType(Context->VoidTy);
   FieldNames[1] = "__gr_top";
 
   // void *__vr_top;
   FieldTypes[2] = Context->getPointerType(Context->VoidTy);
   FieldNames[2] = "__vr_top";
 
   // int __gr_offs;
   FieldTypes[3] = Context->IntTy;
   FieldNames[3] = "__gr_offs";
 
   // int __vr_offs;
   FieldTypes[4] = Context->IntTy;
   FieldNames[4] = "__vr_offs";
 
   // Create fields
   for (unsigned i = 0; i < NumFields; ++i) {
     FieldDecl *Field = FieldDecl::Create(const_cast<ASTContext &>(*Context),
                                          VaListTagDecl,
                                          SourceLocation(),
                                          SourceLocation(),
                                          &Context->Idents.get(FieldNames[i]),
                                          FieldTypes[i], /*TInfo=*/nullptr,
                                          /*BitWidth=*/nullptr,
                                          /*Mutable=*/false,
                                          ICIS_NoInit);
     Field->setAccess(AS_public);
     VaListTagDecl->addDecl(Field);
   }
   VaListTagDecl->completeDefinition();
   Context->VaListTagDecl = VaListTagDecl;
   QualType VaListTagType = Context->getRecordType(VaListTagDecl);
 
   // } __builtin_va_list;
   return Context->buildImplicitTypedef(VaListTagType, "__builtin_va_list");
 }
 
 static TypedefDecl *CreatePowerABIBuiltinVaListDecl(const ASTContext *Context) {
   // typedef struct __va_list_tag {
   RecordDecl *VaListTagDecl;
 
   VaListTagDecl = Context->buildImplicitRecord("__va_list_tag");
   VaListTagDecl->startDefinition();
 
   const size_t NumFields = 5;
   QualType FieldTypes[NumFields];
   const char *FieldNames[NumFields];
 
   //   unsigned char gpr;
   FieldTypes[0] = Context->UnsignedCharTy;
   FieldNames[0] = "gpr";
 
   //   unsigned char fpr;
   FieldTypes[1] = Context->UnsignedCharTy;
   FieldNames[1] = "fpr";
 
   //   unsigned short reserved;
   FieldTypes[2] = Context->UnsignedShortTy;
   FieldNames[2] = "reserved";
 
   //   void* overflow_arg_area;
   FieldTypes[3] = Context->getPointerType(Context->VoidTy);
   FieldNames[3] = "overflow_arg_area";
 
   //   void* reg_save_area;
   FieldTypes[4] = Context->getPointerType(Context->VoidTy);
   FieldNames[4] = "reg_save_area";
 
   // Create fields
   for (unsigned i = 0; i < NumFields; ++i) {
     FieldDecl *Field = FieldDecl::Create(*Context, VaListTagDecl,
                                          SourceLocation(),
                                          SourceLocation(),
                                          &Context->Idents.get(FieldNames[i]),
                                          FieldTypes[i], /*TInfo=*/nullptr,
                                          /*BitWidth=*/nullptr,
                                          /*Mutable=*/false,
                                          ICIS_NoInit);
     Field->setAccess(AS_public);
     VaListTagDecl->addDecl(Field);
   }
   VaListTagDecl->completeDefinition();
   Context->VaListTagDecl = VaListTagDecl;
   QualType VaListTagType = Context->getRecordType(VaListTagDecl);
 
   // } __va_list_tag;
   TypedefDecl *VaListTagTypedefDecl =
       Context->buildImplicitTypedef(VaListTagType, "__va_list_tag");
 
   QualType VaListTagTypedefType =
     Context->getTypedefType(VaListTagTypedefDecl);
 
   // typedef __va_list_tag __builtin_va_list[1];
   llvm::APInt Size(Context->getTypeSize(Context->getSizeType()), 1);
   QualType VaListTagArrayType
     = Context->getConstantArrayType(VaListTagTypedefType,
                                     Size, nullptr, ArrayType::Normal, 0);
   return Context->buildImplicitTypedef(VaListTagArrayType, "__builtin_va_list");
 }
 
 static TypedefDecl *
 CreateX86_64ABIBuiltinVaListDecl(const ASTContext *Context) {
   // struct __va_list_tag {
   RecordDecl *VaListTagDecl;
   VaListTagDecl = Context->buildImplicitRecord("__va_list_tag");
   VaListTagDecl->startDefinition();
 
   const size_t NumFields = 4;
   QualType FieldTypes[NumFields];
   const char *FieldNames[NumFields];
 
   //   unsigned gp_offset;
   FieldTypes[0] = Context->UnsignedIntTy;
   FieldNames[0] = "gp_offset";
 
   //   unsigned fp_offset;
   FieldTypes[1] = Context->UnsignedIntTy;
   FieldNames[1] = "fp_offset";
 
   //   void* overflow_arg_area;
   FieldTypes[2] = Context->getPointerType(Context->VoidTy);
   FieldNames[2] = "overflow_arg_area";
 
   //   void* reg_save_area;
   FieldTypes[3] = Context->getPointerType(Context->VoidTy);
   FieldNames[3] = "reg_save_area";
 
   // Create fields
   for (unsigned i = 0; i < NumFields; ++i) {
     FieldDecl *Field = FieldDecl::Create(const_cast<ASTContext &>(*Context),
                                          VaListTagDecl,
                                          SourceLocation(),
                                          SourceLocation(),
                                          &Context->Idents.get(FieldNames[i]),
                                          FieldTypes[i], /*TInfo=*/nullptr,
                                          /*BitWidth=*/nullptr,
                                          /*Mutable=*/false,
                                          ICIS_NoInit);
     Field->setAccess(AS_public);
     VaListTagDecl->addDecl(Field);
   }
   VaListTagDecl->completeDefinition();
   Context->VaListTagDecl = VaListTagDecl;
   QualType VaListTagType = Context->getRecordType(VaListTagDecl);
 
   // };
 
   // typedef struct __va_list_tag __builtin_va_list[1];
   llvm::APInt Size(Context->getTypeSize(Context->getSizeType()), 1);
   QualType VaListTagArrayType = Context->getConstantArrayType(
       VaListTagType, Size, nullptr, ArrayType::Normal, 0);
   return Context->buildImplicitTypedef(VaListTagArrayType, "__builtin_va_list");
 }
 
 static TypedefDecl *CreatePNaClABIBuiltinVaListDecl(const ASTContext *Context) {
   // typedef int __builtin_va_list[4];
   llvm::APInt Size(Context->getTypeSize(Context->getSizeType()), 4);
   QualType IntArrayType = Context->getConstantArrayType(
       Context->IntTy, Size, nullptr, ArrayType::Normal, 0);
   return Context->buildImplicitTypedef(IntArrayType, "__builtin_va_list");
 }
 
 static TypedefDecl *
 CreateAAPCSABIBuiltinVaListDecl(const ASTContext *Context) {
   // struct __va_list
   RecordDecl *VaListDecl = Context->buildImplicitRecord("__va_list");
   if (Context->getLangOpts().CPlusPlus) {
     // namespace std { struct __va_list {
     NamespaceDecl *NS;
     NS = NamespaceDecl::Create(const_cast<ASTContext &>(*Context),
                                Context->getTranslationUnitDecl(),
                                /*Inline*/false, SourceLocation(),
                                SourceLocation(), &Context->Idents.get("std"),
                                /*PrevDecl*/ nullptr);
     NS->setImplicit();
     VaListDecl->setDeclContext(NS);
   }
 
   VaListDecl->startDefinition();
 
   // void * __ap;
   FieldDecl *Field = FieldDecl::Create(const_cast<ASTContext &>(*Context),
                                        VaListDecl,
                                        SourceLocation(),
                                        SourceLocation(),
                                        &Context->Idents.get("__ap"),
                                        Context->getPointerType(Context->VoidTy),
                                        /*TInfo=*/nullptr,
                                        /*BitWidth=*/nullptr,
                                        /*Mutable=*/false,
                                        ICIS_NoInit);
   Field->setAccess(AS_public);
   VaListDecl->addDecl(Field);
 
   // };
   VaListDecl->completeDefinition();
   Context->VaListTagDecl = VaListDecl;
 
   // typedef struct __va_list __builtin_va_list;
   QualType T = Context->getRecordType(VaListDecl);
   return Context->buildImplicitTypedef(T, "__builtin_va_list");
 }
 
 static TypedefDecl *
 CreateSystemZBuiltinVaListDecl(const ASTContext *Context) {
   // struct __va_list_tag {
   RecordDecl *VaListTagDecl;
   VaListTagDecl = Context->buildImplicitRecord("__va_list_tag");
   VaListTagDecl->startDefinition();
 
   const size_t NumFields = 4;
   QualType FieldTypes[NumFields];
   const char *FieldNames[NumFields];
 
   //   long __gpr;
   FieldTypes[0] = Context->LongTy;
   FieldNames[0] = "__gpr";
 
   //   long __fpr;
   FieldTypes[1] = Context->LongTy;
   FieldNames[1] = "__fpr";
 
   //   void *__overflow_arg_area;
   FieldTypes[2] = Context->getPointerType(Context->VoidTy);
   FieldNames[2] = "__overflow_arg_area";
 
   //   void *__reg_save_area;
   FieldTypes[3] = Context->getPointerType(Context->VoidTy);
   FieldNames[3] = "__reg_save_area";
 
   // Create fields
   for (unsigned i = 0; i < NumFields; ++i) {
     FieldDecl *Field = FieldDecl::Create(const_cast<ASTContext &>(*Context),
                                          VaListTagDecl,
                                          SourceLocation(),
                                          SourceLocation(),
                                          &Context->Idents.get(FieldNames[i]),
                                          FieldTypes[i], /*TInfo=*/nullptr,
                                          /*BitWidth=*/nullptr,
                                          /*Mutable=*/false,
                                          ICIS_NoInit);
     Field->setAccess(AS_public);
     VaListTagDecl->addDecl(Field);
   }
   VaListTagDecl->completeDefinition();
   Context->VaListTagDecl = VaListTagDecl;
   QualType VaListTagType = Context->getRecordType(VaListTagDecl);
 
   // };
 
   // typedef __va_list_tag __builtin_va_list[1];
   llvm::APInt Size(Context->getTypeSize(Context->getSizeType()), 1);
   QualType VaListTagArrayType = Context->getConstantArrayType(
       VaListTagType, Size, nullptr, ArrayType::Normal, 0);
 
   return Context->buildImplicitTypedef(VaListTagArrayType, "__builtin_va_list");
 }
 
 static TypedefDecl *CreateHexagonBuiltinVaListDecl(const ASTContext *Context) {
   // typedef struct __va_list_tag {
   RecordDecl *VaListTagDecl;
   VaListTagDecl = Context->buildImplicitRecord("__va_list_tag");
   VaListTagDecl->startDefinition();
 
   const size_t NumFields = 3;
   QualType FieldTypes[NumFields];
   const char *FieldNames[NumFields];
 
   //   void *CurrentSavedRegisterArea;
   FieldTypes[0] = Context->getPointerType(Context->VoidTy);
   FieldNames[0] = "__current_saved_reg_area_pointer";
 
   //   void *SavedRegAreaEnd;
   FieldTypes[1] = Context->getPointerType(Context->VoidTy);
   FieldNames[1] = "__saved_reg_area_end_pointer";
 
   //   void *OverflowArea;
   FieldTypes[2] = Context->getPointerType(Context->VoidTy);
   FieldNames[2] = "__overflow_area_pointer";
 
   // Create fields
   for (unsigned i = 0; i < NumFields; ++i) {
     FieldDecl *Field = FieldDecl::Create(
         const_cast<ASTContext &>(*Context), VaListTagDecl, SourceLocation(),
         SourceLocation(), &Context->Idents.get(FieldNames[i]), FieldTypes[i],
         /*TInfo=*/nullptr,
         /*BitWidth=*/nullptr,
         /*Mutable=*/false, ICIS_NoInit);
     Field->setAccess(AS_public);
     VaListTagDecl->addDecl(Field);
   }
   VaListTagDecl->completeDefinition();
   Context->VaListTagDecl = VaListTagDecl;
   QualType VaListTagType = Context->getRecordType(VaListTagDecl);
 
   // } __va_list_tag;
   TypedefDecl *VaListTagTypedefDecl =
       Context->buildImplicitTypedef(VaListTagType, "__va_list_tag");
 
   QualType VaListTagTypedefType = Context->getTypedefType(VaListTagTypedefDecl);
 
   // typedef __va_list_tag __builtin_va_list[1];
   llvm::APInt Size(Context->getTypeSize(Context->getSizeType()), 1);
   QualType VaListTagArrayType = Context->getConstantArrayType(
       VaListTagTypedefType, Size, nullptr, ArrayType::Normal, 0);
 
   return Context->buildImplicitTypedef(VaListTagArrayType, "__builtin_va_list");
 }
 
 static TypedefDecl *CreateVaListDecl(const ASTContext *Context,
                                      TargetInfo::BuiltinVaListKind Kind) {
   switch (Kind) {
   case TargetInfo::CharPtrBuiltinVaList:
     return CreateCharPtrBuiltinVaListDecl(Context);
   case TargetInfo::VoidPtrBuiltinVaList:
     return CreateVoidPtrBuiltinVaListDecl(Context);
   case TargetInfo::AArch64ABIBuiltinVaList:
     return CreateAArch64ABIBuiltinVaListDecl(Context);
   case TargetInfo::PowerABIBuiltinVaList:
     return CreatePowerABIBuiltinVaListDecl(Context);
   case TargetInfo::X86_64ABIBuiltinVaList:
     return CreateX86_64ABIBuiltinVaListDecl(Context);
   case TargetInfo::PNaClABIBuiltinVaList:
     return CreatePNaClABIBuiltinVaListDecl(Context);
   case TargetInfo::AAPCSABIBuiltinVaList:
     return CreateAAPCSABIBuiltinVaListDecl(Context);
   case TargetInfo::SystemZBuiltinVaList:
     return CreateSystemZBuiltinVaListDecl(Context);
   case TargetInfo::HexagonBuiltinVaList:
     return CreateHexagonBuiltinVaListDecl(Context);
   }
 
   llvm_unreachable("Unhandled __builtin_va_list type kind");
 }
 
 TypedefDecl *ASTContext::getBuiltinVaListDecl() const {
   if (!BuiltinVaListDecl) {
     BuiltinVaListDecl = CreateVaListDecl(this, Target->getBuiltinVaListKind());
     assert(BuiltinVaListDecl->isImplicit());
   }
 
   return BuiltinVaListDecl;
 }
 
 Decl *ASTContext::getVaListTagDecl() const {
   // Force the creation of VaListTagDecl by building the __builtin_va_list
   // declaration.
   if (!VaListTagDecl)
     (void)getBuiltinVaListDecl();
 
   return VaListTagDecl;
 }
 
 TypedefDecl *ASTContext::getBuiltinMSVaListDecl() const {
   if (!BuiltinMSVaListDecl)
     BuiltinMSVaListDecl = CreateMSVaListDecl(this);
 
   return BuiltinMSVaListDecl;
 }
 
 bool ASTContext::canBuiltinBeRedeclared(const FunctionDecl *FD) const {
   return BuiltinInfo.canBeRedeclared(FD->getBuiltinID());
 }
 
 void ASTContext::setObjCConstantStringInterface(ObjCInterfaceDecl *Decl) {
   assert(ObjCConstantStringType.isNull() &&
          "'NSConstantString' type already set!");
 
   ObjCConstantStringType = getObjCInterfaceType(Decl);
 }
 
 /// Retrieve the template name that corresponds to a non-empty
 /// lookup.
 TemplateName
 ASTContext::getOverloadedTemplateName(UnresolvedSetIterator Begin,
                                       UnresolvedSetIterator End) const {
   unsigned size = End - Begin;
   assert(size > 1 && "set is not overloaded!");
 
   void *memory = Allocate(sizeof(OverloadedTemplateStorage) +
                           size * sizeof(FunctionTemplateDecl*));
   auto *OT = new (memory) OverloadedTemplateStorage(size);
 
   NamedDecl **Storage = OT->getStorage();
   for (UnresolvedSetIterator I = Begin; I != End; ++I) {
     NamedDecl *D = *I;
     assert(isa<FunctionTemplateDecl>(D) ||
            isa<UnresolvedUsingValueDecl>(D) ||
            (isa<UsingShadowDecl>(D) &&
             isa<FunctionTemplateDecl>(D->getUnderlyingDecl())));
     *Storage++ = D;
   }
 
   return TemplateName(OT);
 }
 
 /// Retrieve a template name representing an unqualified-id that has been
 /// assumed to name a template for ADL purposes.
 TemplateName ASTContext::getAssumedTemplateName(DeclarationName Name) const {
   auto *OT = new (*this) AssumedTemplateStorage(Name);
   return TemplateName(OT);
 }
 
 /// Retrieve the template name that represents a qualified
 /// template name such as \c std::vector.
 TemplateName
 ASTContext::getQualifiedTemplateName(NestedNameSpecifier *NNS,
                                      bool TemplateKeyword,
                                      TemplateDecl *Template) const {
   assert(NNS && "Missing nested-name-specifier in qualified template name");
 
   // FIXME: Canonicalization?
   llvm::FoldingSetNodeID ID;
   QualifiedTemplateName::Profile(ID, NNS, TemplateKeyword, Template);
 
   void *InsertPos = nullptr;
   QualifiedTemplateName *QTN =
     QualifiedTemplateNames.FindNodeOrInsertPos(ID, InsertPos);
   if (!QTN) {
     QTN = new (*this, alignof(QualifiedTemplateName))
         QualifiedTemplateName(NNS, TemplateKeyword, Template);
     QualifiedTemplateNames.InsertNode(QTN, InsertPos);
   }
 
   return TemplateName(QTN);
 }
 
 /// Retrieve the template name that represents a dependent
 /// template name such as \c MetaFun::template apply.
 TemplateName
 ASTContext::getDependentTemplateName(NestedNameSpecifier *NNS,
                                      const IdentifierInfo *Name) const {
   assert((!NNS || NNS->isDependent()) &&
          "Nested name specifier must be dependent");
 
   llvm::FoldingSetNodeID ID;
   DependentTemplateName::Profile(ID, NNS, Name);
 
   void *InsertPos = nullptr;
   DependentTemplateName *QTN =
     DependentTemplateNames.FindNodeOrInsertPos(ID, InsertPos);
 
   if (QTN)
     return TemplateName(QTN);
 
   NestedNameSpecifier *CanonNNS = getCanonicalNestedNameSpecifier(NNS);
   if (CanonNNS == NNS) {
     QTN = new (*this, alignof(DependentTemplateName))
         DependentTemplateName(NNS, Name);
   } else {
     TemplateName Canon = getDependentTemplateName(CanonNNS, Name);
     QTN = new (*this, alignof(DependentTemplateName))
         DependentTemplateName(NNS, Name, Canon);
     DependentTemplateName *CheckQTN =
       DependentTemplateNames.FindNodeOrInsertPos(ID, InsertPos);
     assert(!CheckQTN && "Dependent type name canonicalization broken");
     (void)CheckQTN;
   }
 
   DependentTemplateNames.InsertNode(QTN, InsertPos);
   return TemplateName(QTN);
 }
 
 /// Retrieve the template name that represents a dependent
 /// template name such as \c MetaFun::template operator+.
 TemplateName
 ASTContext::getDependentTemplateName(NestedNameSpecifier *NNS,
                                      OverloadedOperatorKind Operator) const {
   assert((!NNS || NNS->isDependent()) &&
          "Nested name specifier must be dependent");
 
   llvm::FoldingSetNodeID ID;
   DependentTemplateName::Profile(ID, NNS, Operator);
 
   void *InsertPos = nullptr;
   DependentTemplateName *QTN
     = DependentTemplateNames.FindNodeOrInsertPos(ID, InsertPos);
 
   if (QTN)
     return TemplateName(QTN);
 
   NestedNameSpecifier *CanonNNS = getCanonicalNestedNameSpecifier(NNS);
   if (CanonNNS == NNS) {
     QTN = new (*this, alignof(DependentTemplateName))
         DependentTemplateName(NNS, Operator);
   } else {
     TemplateName Canon = getDependentTemplateName(CanonNNS, Operator);
     QTN = new (*this, alignof(DependentTemplateName))
         DependentTemplateName(NNS, Operator, Canon);
 
     DependentTemplateName *CheckQTN
       = DependentTemplateNames.FindNodeOrInsertPos(ID, InsertPos);
     assert(!CheckQTN && "Dependent template name canonicalization broken");
     (void)CheckQTN;
   }
 
   DependentTemplateNames.InsertNode(QTN, InsertPos);
   return TemplateName(QTN);
 }
 
 TemplateName
 ASTContext::getSubstTemplateTemplateParm(TemplateTemplateParmDecl *param,
                                          TemplateName replacement) const {
   llvm::FoldingSetNodeID ID;
   SubstTemplateTemplateParmStorage::Profile(ID, param, replacement);
 
   void *insertPos = nullptr;
   SubstTemplateTemplateParmStorage *subst
     = SubstTemplateTemplateParms.FindNodeOrInsertPos(ID, insertPos);
 
   if (!subst) {
     subst = new (*this) SubstTemplateTemplateParmStorage(param, replacement);
     SubstTemplateTemplateParms.InsertNode(subst, insertPos);
   }
 
   return TemplateName(subst);
 }
 
 TemplateName
 ASTContext::getSubstTemplateTemplateParmPack(TemplateTemplateParmDecl *Param,
                                        const TemplateArgument &ArgPack) const {
   auto &Self = const_cast<ASTContext &>(*this);
   llvm::FoldingSetNodeID ID;
   SubstTemplateTemplateParmPackStorage::Profile(ID, Self, Param, ArgPack);
 
   void *InsertPos = nullptr;
   SubstTemplateTemplateParmPackStorage *Subst
     = SubstTemplateTemplateParmPacks.FindNodeOrInsertPos(ID, InsertPos);
 
   if (!Subst) {
     Subst = new (*this) SubstTemplateTemplateParmPackStorage(Param,
                                                            ArgPack.pack_size(),
                                                          ArgPack.pack_begin());
     SubstTemplateTemplateParmPacks.InsertNode(Subst, InsertPos);
   }
 
   return TemplateName(Subst);
 }
 
 /// getFromTargetType - Given one of the integer types provided by
 /// TargetInfo, produce the corresponding type. The unsigned @p Type
 /// is actually a value of type @c TargetInfo::IntType.
 CanQualType ASTContext::getFromTargetType(unsigned Type) const {
   switch (Type) {
   case TargetInfo::NoInt: return {};
   case TargetInfo::SignedChar: return SignedCharTy;
   case TargetInfo::UnsignedChar: return UnsignedCharTy;
   case TargetInfo::SignedShort: return ShortTy;
   case TargetInfo::UnsignedShort: return UnsignedShortTy;
   case TargetInfo::SignedInt: return IntTy;
   case TargetInfo::UnsignedInt: return UnsignedIntTy;
   case TargetInfo::SignedLong: return LongTy;
   case TargetInfo::UnsignedLong: return UnsignedLongTy;
   case TargetInfo::SignedLongLong: return LongLongTy;
   case TargetInfo::UnsignedLongLong: return UnsignedLongLongTy;
   }
 
   llvm_unreachable("Unhandled TargetInfo::IntType value");
 }
 
 //===----------------------------------------------------------------------===//
 //                        Type Predicates.
 //===----------------------------------------------------------------------===//
 
 /// getObjCGCAttr - Returns one of GCNone, Weak or Strong objc's
 /// garbage collection attribute.
 ///
 Qualifiers::GC ASTContext::getObjCGCAttrKind(QualType Ty) const {
   if (getLangOpts().getGC() == LangOptions::NonGC)
     return Qualifiers::GCNone;
 
   assert(getLangOpts().ObjC);
   Qualifiers::GC GCAttrs = Ty.getObjCGCAttr();
 
   // Default behaviour under objective-C's gc is for ObjC pointers
   // (or pointers to them) be treated as though they were declared
   // as __strong.
   if (GCAttrs == Qualifiers::GCNone) {
     if (Ty->isObjCObjectPointerType() || Ty->isBlockPointerType())
       return Qualifiers::Strong;
     else if (Ty->isPointerType())
       return getObjCGCAttrKind(Ty->castAs<PointerType>()->getPointeeType());
   } else {
     // It's not valid to set GC attributes on anything that isn't a
     // pointer.
 #ifndef NDEBUG
     QualType CT = Ty->getCanonicalTypeInternal();
     while (const auto *AT = dyn_cast<ArrayType>(CT))
       CT = AT->getElementType();
     assert(CT->isAnyPointerType() || CT->isBlockPointerType());
 #endif
   }
   return GCAttrs;
 }
 
 //===----------------------------------------------------------------------===//
 //                        Type Compatibility Testing
 //===----------------------------------------------------------------------===//
 
 /// areCompatVectorTypes - Return true if the two specified vector types are
 /// compatible.
 static bool areCompatVectorTypes(const VectorType *LHS,
                                  const VectorType *RHS) {
   assert(LHS->isCanonicalUnqualified() && RHS->isCanonicalUnqualified());
   return LHS->getElementType() == RHS->getElementType() &&
          LHS->getNumElements() == RHS->getNumElements();
 }
 
 /// areCompatMatrixTypes - Return true if the two specified matrix types are
 /// compatible.
 static bool areCompatMatrixTypes(const ConstantMatrixType *LHS,
                                  const ConstantMatrixType *RHS) {
   assert(LHS->isCanonicalUnqualified() && RHS->isCanonicalUnqualified());
   return LHS->getElementType() == RHS->getElementType() &&
          LHS->getNumRows() == RHS->getNumRows() &&
          LHS->getNumColumns() == RHS->getNumColumns();
 }
 
 bool ASTContext::areCompatibleVectorTypes(QualType FirstVec,
                                           QualType SecondVec) {
   assert(FirstVec->isVectorType() && "FirstVec should be a vector type");
   assert(SecondVec->isVectorType() && "SecondVec should be a vector type");
 
   if (hasSameUnqualifiedType(FirstVec, SecondVec))
     return true;
 
   // Treat Neon vector types and most AltiVec vector types as if they are the
   // equivalent GCC vector types.
   const auto *First = FirstVec->castAs<VectorType>();
   const auto *Second = SecondVec->castAs<VectorType>();
   if (First->getNumElements() == Second->getNumElements() &&
       hasSameType(First->getElementType(), Second->getElementType()) &&
       First->getVectorKind() != VectorType::AltiVecPixel &&
       First->getVectorKind() != VectorType::AltiVecBool &&
       Second->getVectorKind() != VectorType::AltiVecPixel &&
       Second->getVectorKind() != VectorType::AltiVecBool &&
       First->getVectorKind() != VectorType::SveFixedLengthDataVector &&
       First->getVectorKind() != VectorType::SveFixedLengthPredicateVector &&
       Second->getVectorKind() != VectorType::SveFixedLengthDataVector &&
       Second->getVectorKind() != VectorType::SveFixedLengthPredicateVector)
     return true;
 
   return false;
 }
 
 /// getSVETypeSize - Return SVE vector or predicate register size.
 static uint64_t getSVETypeSize(ASTContext &Context, const BuiltinType *Ty) {
   assert(Ty->isVLSTBuiltinType() && "Invalid SVE Type");
   return Ty->getKind() == BuiltinType::SveBool
              ? (Context.getLangOpts().VScaleMin * 128) / Context.getCharWidth()
              : Context.getLangOpts().VScaleMin * 128;
 }
 
 bool ASTContext::areCompatibleSveTypes(QualType FirstType,
                                        QualType SecondType) {
   assert(((FirstType->isSizelessBuiltinType() && SecondType->isVectorType()) ||
           (FirstType->isVectorType() && SecondType->isSizelessBuiltinType())) &&
          "Expected SVE builtin type and vector type!");
 
   auto IsValidCast = [this](QualType FirstType, QualType SecondType) {
     if (const auto *BT = FirstType->getAs<BuiltinType>()) {
       if (const auto *VT = SecondType->getAs<VectorType>()) {
         // Predicates have the same representation as uint8 so we also have to
         // check the kind to make these types incompatible.
         if (VT->getVectorKind() == VectorType::SveFixedLengthPredicateVector)
           return BT->getKind() == BuiltinType::SveBool;
         else if (VT->getVectorKind() == VectorType::SveFixedLengthDataVector)
           return VT->getElementType().getCanonicalType() ==
                  FirstType->getSveEltType(*this);
         else if (VT->getVectorKind() == VectorType::GenericVector)
           return getTypeSize(SecondType) == getSVETypeSize(*this, BT) &&
                  hasSameType(VT->getElementType(),
                              getBuiltinVectorTypeInfo(BT).ElementType);
       }
     }
     return false;
   };
 
   return IsValidCast(FirstType, SecondType) ||
          IsValidCast(SecondType, FirstType);
 }
 
 bool ASTContext::areLaxCompatibleSveTypes(QualType FirstType,
                                           QualType SecondType) {
   assert(((FirstType->isSizelessBuiltinType() && SecondType->isVectorType()) ||
           (FirstType->isVectorType() && SecondType->isSizelessBuiltinType())) &&
          "Expected SVE builtin type and vector type!");
 
   auto IsLaxCompatible = [this](QualType FirstType, QualType SecondType) {
     const auto *BT = FirstType->getAs<BuiltinType>();
     if (!BT)
       return false;
 
     const auto *VecTy = SecondType->getAs<VectorType>();
     if (VecTy &&
         (VecTy->getVectorKind() == VectorType::SveFixedLengthDataVector ||
          VecTy->getVectorKind() == VectorType::GenericVector)) {
       const LangOptions::LaxVectorConversionKind LVCKind =
           getLangOpts().getLaxVectorConversions();
 
       // Can not convert between sve predicates and sve vectors because of
       // different size.
       if (BT->getKind() == BuiltinType::SveBool &&
           VecTy->getVectorKind() == VectorType::SveFixedLengthDataVector)
         return false;
 
       // If __ARM_FEATURE_SVE_BITS != N do not allow GNU vector lax conversion.
       // "Whenever __ARM_FEATURE_SVE_BITS==N, GNUT implicitly
       // converts to VLAT and VLAT implicitly converts to GNUT."
       // ACLE Spec Version 00bet6, 3.7.3.2. Behavior common to vectors and
       // predicates.
       if (VecTy->getVectorKind() == VectorType::GenericVector &&
           getTypeSize(SecondType) != getSVETypeSize(*this, BT))
         return false;
 
       // If -flax-vector-conversions=all is specified, the types are
       // certainly compatible.
       if (LVCKind == LangOptions::LaxVectorConversionKind::All)
         return true;
 
       // If -flax-vector-conversions=integer is specified, the types are
       // compatible if the elements are integer types.
       if (LVCKind == LangOptions::LaxVectorConversionKind::Integer)
         return VecTy->getElementType().getCanonicalType()->isIntegerType() &&
                FirstType->getSveEltType(*this)->isIntegerType();
     }
 
     return false;
   };
 
   return IsLaxCompatible(FirstType, SecondType) ||
          IsLaxCompatible(SecondType, FirstType);
 }
 
 bool ASTContext::hasDirectOwnershipQualifier(QualType Ty) const {
   while (true) {
     // __strong id
     if (const AttributedType *Attr = dyn_cast<AttributedType>(Ty)) {
       if (Attr->getAttrKind() == attr::ObjCOwnership)
         return true;
 
       Ty = Attr->getModifiedType();
 
     // X *__strong (...)
     } else if (const ParenType *Paren = dyn_cast<ParenType>(Ty)) {
       Ty = Paren->getInnerType();
 
     // We do not want to look through typedefs, typeof(expr),
     // typeof(type), or any other way that the type is somehow
     // abstracted.
     } else {
       return false;
     }
   }
 }
 
 //===----------------------------------------------------------------------===//
 // ObjCQualifiedIdTypesAreCompatible - Compatibility testing for qualified id's.
 //===----------------------------------------------------------------------===//
 
 /// ProtocolCompatibleWithProtocol - return 'true' if 'lProto' is in the
 /// inheritance hierarchy of 'rProto'.
 bool
 ASTContext::ProtocolCompatibleWithProtocol(ObjCProtocolDecl *lProto,
                                            ObjCProtocolDecl *rProto) const {
   if (declaresSameEntity(lProto, rProto))
     return true;
   for (auto *PI : rProto->protocols())
     if (ProtocolCompatibleWithProtocol(lProto, PI))
       return true;
   return false;
 }
 
 /// ObjCQualifiedClassTypesAreCompatible - compare  Class<pr,...> and
 /// Class<pr1, ...>.
 bool ASTContext::ObjCQualifiedClassTypesAreCompatible(
     const ObjCObjectPointerType *lhs, const ObjCObjectPointerType *rhs) {
   for (auto *lhsProto : lhs->quals()) {
     bool match = false;
     for (auto *rhsProto : rhs->quals()) {
       if (ProtocolCompatibleWithProtocol(lhsProto, rhsProto)) {
         match = true;
         break;
       }
     }
     if (!match)
       return false;
   }
   return true;
 }
 
 /// ObjCQualifiedIdTypesAreCompatible - We know that one of lhs/rhs is an
 /// ObjCQualifiedIDType.
 bool ASTContext::ObjCQualifiedIdTypesAreCompatible(
     const ObjCObjectPointerType *lhs, const ObjCObjectPointerType *rhs,
     bool compare) {
   // Allow id<P..> and an 'id' in all cases.
   if (lhs->isObjCIdType() || rhs->isObjCIdType())
     return true;
 
   // Don't allow id<P..> to convert to Class or Class<P..> in either direction.
   if (lhs->isObjCClassType() || lhs->isObjCQualifiedClassType() ||
       rhs->isObjCClassType() || rhs->isObjCQualifiedClassType())
     return false;
 
   if (lhs->isObjCQualifiedIdType()) {
     if (rhs->qual_empty()) {
       // If the RHS is a unqualified interface pointer "NSString*",
       // make sure we check the class hierarchy.
       if (ObjCInterfaceDecl *rhsID = rhs->getInterfaceDecl()) {
         for (auto *I : lhs->quals()) {
           // when comparing an id<P> on lhs with a static type on rhs,
           // see if static class implements all of id's protocols, directly or
           // through its super class and categories.
           if (!rhsID->ClassImplementsProtocol(I, true))
             return false;
         }
       }
       // If there are no qualifiers and no interface, we have an 'id'.
       return true;
     }
     // Both the right and left sides have qualifiers.
     for (auto *lhsProto : lhs->quals()) {
       bool match = false;
 
       // when comparing an id<P> on lhs with a static type on rhs,
       // see if static class implements all of id's protocols, directly or
       // through its super class and categories.
       for (auto *rhsProto : rhs->quals()) {
         if (ProtocolCompatibleWithProtocol(lhsProto, rhsProto) ||
             (compare && ProtocolCompatibleWithProtocol(rhsProto, lhsProto))) {
           match = true;
           break;
         }
       }
       // If the RHS is a qualified interface pointer "NSString<P>*",
       // make sure we check the class hierarchy.
       if (ObjCInterfaceDecl *rhsID = rhs->getInterfaceDecl()) {
         for (auto *I : lhs->quals()) {
           // when comparing an id<P> on lhs with a static type on rhs,
           // see if static class implements all of id's protocols, directly or
           // through its super class and categories.
           if (rhsID->ClassImplementsProtocol(I, true)) {
             match = true;
             break;
           }
         }
       }
       if (!match)
         return false;
     }
 
     return true;
   }
 
   assert(rhs->isObjCQualifiedIdType() && "One of the LHS/RHS should be id<x>");
 
   if (lhs->getInterfaceType()) {
     // If both the right and left sides have qualifiers.
     for (auto *lhsProto : lhs->quals()) {
       bool match = false;
 
       // when comparing an id<P> on rhs with a static type on lhs,
       // see if static class implements all of id's protocols, directly or
       // through its super class and categories.
       // First, lhs protocols in the qualifier list must be found, direct
       // or indirect in rhs's qualifier list or it is a mismatch.
       for (auto *rhsProto : rhs->quals()) {
         if (ProtocolCompatibleWithProtocol(lhsProto, rhsProto) ||
             (compare && ProtocolCompatibleWithProtocol(rhsProto, lhsProto))) {
           match = true;
           break;
         }
       }
       if (!match)
         return false;
     }
 
     // Static class's protocols, or its super class or category protocols
     // must be found, direct or indirect in rhs's qualifier list or it is a mismatch.
     if (ObjCInterfaceDecl *lhsID = lhs->getInterfaceDecl()) {
       llvm::SmallPtrSet<ObjCProtocolDecl *, 8> LHSInheritedProtocols;
       CollectInheritedProtocols(lhsID, LHSInheritedProtocols);
       // This is rather dubious but matches gcc's behavior. If lhs has
       // no type qualifier and its class has no static protocol(s)
       // assume that it is mismatch.
       if (LHSInheritedProtocols.empty() && lhs->qual_empty())
         return false;
       for (auto *lhsProto : LHSInheritedProtocols) {
         bool match = false;
         for (auto *rhsProto : rhs->quals()) {
           if (ProtocolCompatibleWithProtocol(lhsProto, rhsProto) ||
               (compare && ProtocolCompatibleWithProtocol(rhsProto, lhsProto))) {
             match = true;
             break;
           }
         }
         if (!match)
           return false;
       }
     }
     return true;
   }
   return false;
 }
 
 /// canAssignObjCInterfaces - Return true if the two interface types are
 /// compatible for assignment from RHS to LHS.  This handles validation of any
 /// protocol qualifiers on the LHS or RHS.
 bool ASTContext::canAssignObjCInterfaces(const ObjCObjectPointerType *LHSOPT,
                                          const ObjCObjectPointerType *RHSOPT) {
   const ObjCObjectType* LHS = LHSOPT->getObjectType();
   const ObjCObjectType* RHS = RHSOPT->getObjectType();
 
   // If either type represents the built-in 'id' type, return true.
   if (LHS->isObjCUnqualifiedId() || RHS->isObjCUnqualifiedId())
     return true;
 
   // Function object that propagates a successful result or handles
   // __kindof types.
   auto finish = [&](bool succeeded) -> bool {
     if (succeeded)
       return true;
 
     if (!RHS->isKindOfType())
       return false;
 
     // Strip off __kindof and protocol qualifiers, then check whether
     // we can assign the other way.
     return canAssignObjCInterfaces(RHSOPT->stripObjCKindOfTypeAndQuals(*this),
                                    LHSOPT->stripObjCKindOfTypeAndQuals(*this));
   };
 
   // Casts from or to id<P> are allowed when the other side has compatible
   // protocols.
   if (LHS->isObjCQualifiedId() || RHS->isObjCQualifiedId()) {
     return finish(ObjCQualifiedIdTypesAreCompatible(LHSOPT, RHSOPT, false));
   }
 
   // Verify protocol compatibility for casts from Class<P1> to Class<P2>.
   if (LHS->isObjCQualifiedClass() && RHS->isObjCQualifiedClass()) {
     return finish(ObjCQualifiedClassTypesAreCompatible(LHSOPT, RHSOPT));
   }
 
   // Casts from Class to Class<Foo>, or vice-versa, are allowed.
   if (LHS->isObjCClass() && RHS->isObjCClass()) {
     return true;
   }
 
   // If we have 2 user-defined types, fall into that path.
   if (LHS->getInterface() && RHS->getInterface()) {
     return finish(canAssignObjCInterfaces(LHS, RHS));
   }
 
   return false;
 }
 
 /// canAssignObjCInterfacesInBlockPointer - This routine is specifically written
 /// for providing type-safety for objective-c pointers used to pass/return
 /// arguments in block literals. When passed as arguments, passing 'A*' where
 /// 'id' is expected is not OK. Passing 'Sub *" where 'Super *" is expected is
 /// not OK. For the return type, the opposite is not OK.
 bool ASTContext::canAssignObjCInterfacesInBlockPointer(
                                          const ObjCObjectPointerType *LHSOPT,
                                          const ObjCObjectPointerType *RHSOPT,
                                          bool BlockReturnType) {
 
   // Function object that propagates a successful result or handles
   // __kindof types.
   auto finish = [&](bool succeeded) -> bool {
     if (succeeded)
       return true;
 
     const ObjCObjectPointerType *Expected = BlockReturnType ? RHSOPT : LHSOPT;
     if (!Expected->isKindOfType())
       return false;
 
     // Strip off __kindof and protocol qualifiers, then check whether
     // we can assign the other way.
     return canAssignObjCInterfacesInBlockPointer(
              RHSOPT->stripObjCKindOfTypeAndQuals(*this),
              LHSOPT->stripObjCKindOfTypeAndQuals(*this),
              BlockReturnType);
   };
 
   if (RHSOPT->isObjCBuiltinType() || LHSOPT->isObjCIdType())
     return true;
 
   if (LHSOPT->isObjCBuiltinType()) {
     return finish(RHSOPT->isObjCBuiltinType() ||
                   RHSOPT->isObjCQualifiedIdType());
   }
 
   if (LHSOPT->isObjCQualifiedIdType() || RHSOPT->isObjCQualifiedIdType()) {
     if (getLangOpts().CompatibilityQualifiedIdBlockParamTypeChecking)
       // Use for block parameters previous type checking for compatibility.
       return finish(ObjCQualifiedIdTypesAreCompatible(LHSOPT, RHSOPT, false) ||
                     // Or corrected type checking as in non-compat mode.
                     (!BlockReturnType &&
                      ObjCQualifiedIdTypesAreCompatible(RHSOPT, LHSOPT, false)));
     else
       return finish(ObjCQualifiedIdTypesAreCompatible(
           (BlockReturnType ? LHSOPT : RHSOPT),
           (BlockReturnType ? RHSOPT : LHSOPT), false));
   }
 
   const ObjCInterfaceType* LHS = LHSOPT->getInterfaceType();
   const ObjCInterfaceType* RHS = RHSOPT->getInterfaceType();
   if (LHS && RHS)  { // We have 2 user-defined types.
     if (LHS != RHS) {
       if (LHS->getDecl()->isSuperClassOf(RHS->getDecl()))
         return finish(BlockReturnType);
       if (RHS->getDecl()->isSuperClassOf(LHS->getDecl()))
         return finish(!BlockReturnType);
     }
     else
       return true;
   }
   return false;
 }
 
 /// Comparison routine for Objective-C protocols to be used with
 /// llvm::array_pod_sort.
 static int compareObjCProtocolsByName(ObjCProtocolDecl * const *lhs,
                                       ObjCProtocolDecl * const *rhs) {
   return (*lhs)->getName().compare((*rhs)->getName());
 }
 
 /// getIntersectionOfProtocols - This routine finds the intersection of set
 /// of protocols inherited from two distinct objective-c pointer objects with
 /// the given common base.
 /// It is used to build composite qualifier list of the composite type of
 /// the conditional expression involving two objective-c pointer objects.
 static
 void getIntersectionOfProtocols(ASTContext &Context,
                                 const ObjCInterfaceDecl *CommonBase,
                                 const ObjCObjectPointerType *LHSOPT,
                                 const ObjCObjectPointerType *RHSOPT,
       SmallVectorImpl<ObjCProtocolDecl *> &IntersectionSet) {
 
   const ObjCObjectType* LHS = LHSOPT->getObjectType();
   const ObjCObjectType* RHS = RHSOPT->getObjectType();
   assert(LHS->getInterface() && "LHS must have an interface base");
   assert(RHS->getInterface() && "RHS must have an interface base");
 
   // Add all of the protocols for the LHS.
   llvm::SmallPtrSet<ObjCProtocolDecl *, 8> LHSProtocolSet;
 
   // Start with the protocol qualifiers.
   for (auto proto : LHS->quals()) {
     Context.CollectInheritedProtocols(proto, LHSProtocolSet);
   }
 
   // Also add the protocols associated with the LHS interface.
   Context.CollectInheritedProtocols(LHS->getInterface(), LHSProtocolSet);
 
   // Add all of the protocols for the RHS.
   llvm::SmallPtrSet<ObjCProtocolDecl *, 8> RHSProtocolSet;
 
   // Start with the protocol qualifiers.
   for (auto proto : RHS->quals()) {
     Context.CollectInheritedProtocols(proto, RHSProtocolSet);
   }
 
   // Also add the protocols associated with the RHS interface.
   Context.CollectInheritedProtocols(RHS->getInterface(), RHSProtocolSet);
 
   // Compute the intersection of the collected protocol sets.
   for (auto proto : LHSProtocolSet) {
     if (RHSProtocolSet.count(proto))
       IntersectionSet.push_back(proto);
   }
 
   // Compute the set of protocols that is implied by either the common type or
   // the protocols within the intersection.
   llvm::SmallPtrSet<ObjCProtocolDecl *, 8> ImpliedProtocols;
   Context.CollectInheritedProtocols(CommonBase, ImpliedProtocols);
 
   // Remove any implied protocols from the list of inherited protocols.
   if (!ImpliedProtocols.empty()) {
     llvm::erase_if(IntersectionSet, [&](ObjCProtocolDecl *proto) -> bool {
       return ImpliedProtocols.contains(proto);
     });
   }
 
   // Sort the remaining protocols by name.
   llvm::array_pod_sort(IntersectionSet.begin(), IntersectionSet.end(),
                        compareObjCProtocolsByName);
 }
 
 /// Determine whether the first type is a subtype of the second.
 static bool canAssignObjCObjectTypes(ASTContext &ctx, QualType lhs,
                                      QualType rhs) {
   // Common case: two object pointers.
   const auto *lhsOPT = lhs->getAs<ObjCObjectPointerType>();
   const auto *rhsOPT = rhs->getAs<ObjCObjectPointerType>();
   if (lhsOPT && rhsOPT)
     return ctx.canAssignObjCInterfaces(lhsOPT, rhsOPT);
 
   // Two block pointers.
   const auto *lhsBlock = lhs->getAs<BlockPointerType>();
   const auto *rhsBlock = rhs->getAs<BlockPointerType>();
   if (lhsBlock && rhsBlock)
     return ctx.typesAreBlockPointerCompatible(lhs, rhs);
 
   // If either is an unqualified 'id' and the other is a block, it's
   // acceptable.
   if ((lhsOPT && lhsOPT->isObjCIdType() && rhsBlock) ||
       (rhsOPT && rhsOPT->isObjCIdType() && lhsBlock))
     return true;
 
   return false;
 }
 
 // Check that the given Objective-C type argument lists are equivalent.
 static bool sameObjCTypeArgs(ASTContext &ctx,
                              const ObjCInterfaceDecl *iface,
                              ArrayRef<QualType> lhsArgs,
                              ArrayRef<QualType> rhsArgs,
                              bool stripKindOf) {
   if (lhsArgs.size() != rhsArgs.size())
     return false;
 
   ObjCTypeParamList *typeParams = iface->getTypeParamList();
   for (unsigned i = 0, n = lhsArgs.size(); i != n; ++i) {
     if (ctx.hasSameType(lhsArgs[i], rhsArgs[i]))
       continue;
 
     switch (typeParams->begin()[i]->getVariance()) {
     case ObjCTypeParamVariance::Invariant:
       if (!stripKindOf ||
           !ctx.hasSameType(lhsArgs[i].stripObjCKindOfType(ctx),
                            rhsArgs[i].stripObjCKindOfType(ctx))) {
         return false;
       }
       break;
 
     case ObjCTypeParamVariance::Covariant:
       if (!canAssignObjCObjectTypes(ctx, lhsArgs[i], rhsArgs[i]))
         return false;
       break;
 
     case ObjCTypeParamVariance::Contravariant:
       if (!canAssignObjCObjectTypes(ctx, rhsArgs[i], lhsArgs[i]))
         return false;
       break;
     }
   }
 
   return true;
 }
 
 QualType ASTContext::areCommonBaseCompatible(
            const ObjCObjectPointerType *Lptr,
            const ObjCObjectPointerType *Rptr) {
   const ObjCObjectType *LHS = Lptr->getObjectType();
   const ObjCObjectType *RHS = Rptr->getObjectType();
   const ObjCInterfaceDecl* LDecl = LHS->getInterface();
   const ObjCInterfaceDecl* RDecl = RHS->getInterface();
 
   if (!LDecl || !RDecl)
     return {};
 
   // When either LHS or RHS is a kindof type, we should return a kindof type.
   // For example, for common base of kindof(ASub1) and kindof(ASub2), we return
   // kindof(A).
   bool anyKindOf = LHS->isKindOfType() || RHS->isKindOfType();
 
   // Follow the left-hand side up the class hierarchy until we either hit a
   // root or find the RHS. Record the ancestors in case we don't find it.
   llvm::SmallDenseMap<const ObjCInterfaceDecl *, const ObjCObjectType *, 4>
     LHSAncestors;
   while (true) {
     // Record this ancestor. We'll need this if the common type isn't in the
     // path from the LHS to the root.
     LHSAncestors[LHS->getInterface()->getCanonicalDecl()] = LHS;
 
     if (declaresSameEntity(LHS->getInterface(), RDecl)) {
       // Get the type arguments.
       ArrayRef<QualType> LHSTypeArgs = LHS->getTypeArgsAsWritten();
       bool anyChanges = false;
       if (LHS->isSpecialized() && RHS->isSpecialized()) {
         // Both have type arguments, compare them.
         if (!sameObjCTypeArgs(*this, LHS->getInterface(),
                               LHS->getTypeArgs(), RHS->getTypeArgs(),
                               /*stripKindOf=*/true))
           return {};
       } else if (LHS->isSpecialized() != RHS->isSpecialized()) {
         // If only one has type arguments, the result will not have type
         // arguments.
         LHSTypeArgs = {};
         anyChanges = true;
       }
 
       // Compute the intersection of protocols.
       SmallVector<ObjCProtocolDecl *, 8> Protocols;
       getIntersectionOfProtocols(*this, LHS->getInterface(), Lptr, Rptr,
                                  Protocols);
       if (!Protocols.empty())
         anyChanges = true;
 
       // If anything in the LHS will have changed, build a new result type.
       // If we need to return a kindof type but LHS is not a kindof type, we
       // build a new result type.
       if (anyChanges || LHS->isKindOfType() != anyKindOf) {
         QualType Result = getObjCInterfaceType(LHS->getInterface());
         Result = getObjCObjectType(Result, LHSTypeArgs, Protocols,
                                    anyKindOf || LHS->isKindOfType());
         return getObjCObjectPointerType(Result);
       }
 
       return getObjCObjectPointerType(QualType(LHS, 0));
     }
 
     // Find the superclass.
     QualType LHSSuperType = LHS->getSuperClassType();
     if (LHSSuperType.isNull())
       break;
 
     LHS = LHSSuperType->castAs<ObjCObjectType>();
   }
 
   // We didn't find anything by following the LHS to its root; now check
   // the RHS against the cached set of ancestors.
   while (true) {
     auto KnownLHS = LHSAncestors.find(RHS->getInterface()->getCanonicalDecl());
     if (KnownLHS != LHSAncestors.end()) {
       LHS = KnownLHS->second;
 
       // Get the type arguments.
       ArrayRef<QualType> RHSTypeArgs = RHS->getTypeArgsAsWritten();
       bool anyChanges = false;
       if (LHS->isSpecialized() && RHS->isSpecialized()) {
         // Both have type arguments, compare them.
         if (!sameObjCTypeArgs(*this, LHS->getInterface(),
                               LHS->getTypeArgs(), RHS->getTypeArgs(),
                               /*stripKindOf=*/true))
           return {};
       } else if (LHS->isSpecialized() != RHS->isSpecialized()) {
         // If only one has type arguments, the result will not have type
         // arguments.
         RHSTypeArgs = {};
         anyChanges = true;
       }
 
       // Compute the intersection of protocols.
       SmallVector<ObjCProtocolDecl *, 8> Protocols;
       getIntersectionOfProtocols(*this, RHS->getInterface(), Lptr, Rptr,
                                  Protocols);
       if (!Protocols.empty())
         anyChanges = true;
 
       // If we need to return a kindof type but RHS is not a kindof type, we
       // build a new result type.
       if (anyChanges || RHS->isKindOfType() != anyKindOf) {
         QualType Result = getObjCInterfaceType(RHS->getInterface());
         Result = getObjCObjectType(Result, RHSTypeArgs, Protocols,
                                    anyKindOf || RHS->isKindOfType());
         return getObjCObjectPointerType(Result);
       }
 
       return getObjCObjectPointerType(QualType(RHS, 0));
     }
 
     // Find the superclass of the RHS.
     QualType RHSSuperType = RHS->getSuperClassType();
     if (RHSSuperType.isNull())
       break;
 
     RHS = RHSSuperType->castAs<ObjCObjectType>();
   }
 
   return {};
 }
 
 bool ASTContext::canAssignObjCInterfaces(const ObjCObjectType *LHS,
                                          const ObjCObjectType *RHS) {
   assert(LHS->getInterface() && "LHS is not an interface type");
   assert(RHS->getInterface() && "RHS is not an interface type");
 
   // Verify that the base decls are compatible: the RHS must be a subclass of
   // the LHS.
   ObjCInterfaceDecl *LHSInterface = LHS->getInterface();
   bool IsSuperClass = LHSInterface->isSuperClassOf(RHS->getInterface());
   if (!IsSuperClass)
     return false;
 
   // If the LHS has protocol qualifiers, determine whether all of them are
   // satisfied by the RHS (i.e., the RHS has a superset of the protocols in the
   // LHS).
   if (LHS->getNumProtocols() > 0) {
     // OK if conversion of LHS to SuperClass results in narrowing of types
     // ; i.e., SuperClass may implement at least one of the protocols
     // in LHS's protocol list. Example, SuperObj<P1> = lhs<P1,P2> is ok.
     // But not SuperObj<P1,P2,P3> = lhs<P1,P2>.
     llvm::SmallPtrSet<ObjCProtocolDecl *, 8> SuperClassInheritedProtocols;
     CollectInheritedProtocols(RHS->getInterface(), SuperClassInheritedProtocols);
     // Also, if RHS has explicit quelifiers, include them for comparing with LHS's
     // qualifiers.
     for (auto *RHSPI : RHS->quals())
       CollectInheritedProtocols(RHSPI, SuperClassInheritedProtocols);
     // If there is no protocols associated with RHS, it is not a match.
     if (SuperClassInheritedProtocols.empty())
       return false;
 
     for (const auto *LHSProto : LHS->quals()) {
       bool SuperImplementsProtocol = false;
       for (auto *SuperClassProto : SuperClassInheritedProtocols)
         if (SuperClassProto->lookupProtocolNamed(LHSProto->getIdentifier())) {
           SuperImplementsProtocol = true;
           break;
         }
       if (!SuperImplementsProtocol)
         return false;
     }
   }
 
   // If the LHS is specialized, we may need to check type arguments.
   if (LHS->isSpecialized()) {
     // Follow the superclass chain until we've matched the LHS class in the
     // hierarchy. This substitutes type arguments through.
     const ObjCObjectType *RHSSuper = RHS;
     while (!declaresSameEntity(RHSSuper->getInterface(), LHSInterface))
       RHSSuper = RHSSuper->getSuperClassType()->castAs<ObjCObjectType>();
 
     // If the RHS is specializd, compare type arguments.
     if (RHSSuper->isSpecialized() &&
         !sameObjCTypeArgs(*this, LHS->getInterface(),
                           LHS->getTypeArgs(), RHSSuper->getTypeArgs(),
                           /*stripKindOf=*/true)) {
       return false;
     }
   }
 
   return true;
 }
 
 bool ASTContext::areComparableObjCPointerTypes(QualType LHS, QualType RHS) {
   // get the "pointed to" types
   const auto *LHSOPT = LHS->getAs<ObjCObjectPointerType>();
   const auto *RHSOPT = RHS->getAs<ObjCObjectPointerType>();
 
   if (!LHSOPT || !RHSOPT)
     return false;
 
   return canAssignObjCInterfaces(LHSOPT, RHSOPT) ||
          canAssignObjCInterfaces(RHSOPT, LHSOPT);
 }
 
 bool ASTContext::canBindObjCObjectType(QualType To, QualType From) {
   return canAssignObjCInterfaces(
       getObjCObjectPointerType(To)->castAs<ObjCObjectPointerType>(),
       getObjCObjectPointerType(From)->castAs<ObjCObjectPointerType>());
 }
 
 /// typesAreCompatible - C99 6.7.3p9: For two qualified types to be compatible,
 /// both shall have the identically qualified version of a compatible type.
 /// C99 6.2.7p1: Two types have compatible types if their types are the
 /// same. See 6.7.[2,3,5] for additional rules.
 bool ASTContext::typesAreCompatible(QualType LHS, QualType RHS,
                                     bool CompareUnqualified) {
   if (getLangOpts().CPlusPlus)
     return hasSameType(LHS, RHS);
 
   return !mergeTypes(LHS, RHS, false, CompareUnqualified).isNull();
 }
 
 bool ASTContext::propertyTypesAreCompatible(QualType LHS, QualType RHS) {
   return typesAreCompatible(LHS, RHS);
 }
 
 bool ASTContext::typesAreBlockPointerCompatible(QualType LHS, QualType RHS) {
   return !mergeTypes(LHS, RHS, true).isNull();
 }
 
 /// mergeTransparentUnionType - if T is a transparent union type and a member
 /// of T is compatible with SubType, return the merged type, else return
 /// QualType()
 QualType ASTContext::mergeTransparentUnionType(QualType T, QualType SubType,
                                                bool OfBlockPointer,
                                                bool Unqualified) {
   if (const RecordType *UT = T->getAsUnionType()) {
     RecordDecl *UD = UT->getDecl();
     if (UD->hasAttr<TransparentUnionAttr>()) {
       for (const auto *I : UD->fields()) {
         QualType ET = I->getType().getUnqualifiedType();
         QualType MT = mergeTypes(ET, SubType, OfBlockPointer, Unqualified);
         if (!MT.isNull())
           return MT;
       }
     }
   }
 
   return {};
 }
 
 /// mergeFunctionParameterTypes - merge two types which appear as function
 /// parameter types
 QualType ASTContext::mergeFunctionParameterTypes(QualType lhs, QualType rhs,
                                                  bool OfBlockPointer,
                                                  bool Unqualified) {
   // GNU extension: two types are compatible if they appear as a function
   // argument, one of the types is a transparent union type and the other
   // type is compatible with a union member
   QualType lmerge = mergeTransparentUnionType(lhs, rhs, OfBlockPointer,
                                               Unqualified);
   if (!lmerge.isNull())
     return lmerge;
 
   QualType rmerge = mergeTransparentUnionType(rhs, lhs, OfBlockPointer,
                                               Unqualified);
   if (!rmerge.isNull())
     return rmerge;
 
   return mergeTypes(lhs, rhs, OfBlockPointer, Unqualified);
 }
 
 QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs,
                                         bool OfBlockPointer, bool Unqualified,
                                         bool AllowCXX) {
   const auto *lbase = lhs->castAs<FunctionType>();
   const auto *rbase = rhs->castAs<FunctionType>();
   const auto *lproto = dyn_cast<FunctionProtoType>(lbase);
   const auto *rproto = dyn_cast<FunctionProtoType>(rbase);
   bool allLTypes = true;
   bool allRTypes = true;
 
   // Check return type
   QualType retType;
   if (OfBlockPointer) {
     QualType RHS = rbase->getReturnType();
     QualType LHS = lbase->getReturnType();
     bool UnqualifiedResult = Unqualified;
     if (!UnqualifiedResult)
       UnqualifiedResult = (!RHS.hasQualifiers() && LHS.hasQualifiers());
     retType = mergeTypes(LHS, RHS, true, UnqualifiedResult, true);
   }
   else
     retType = mergeTypes(lbase->getReturnType(), rbase->getReturnType(), false,
                          Unqualified);
   if (retType.isNull())
     return {};
 
   if (Unqualified)
     retType = retType.getUnqualifiedType();
 
   CanQualType LRetType = getCanonicalType(lbase->getReturnType());
   CanQualType RRetType = getCanonicalType(rbase->getReturnType());
   if (Unqualified) {
     LRetType = LRetType.getUnqualifiedType();
     RRetType = RRetType.getUnqualifiedType();
   }
 
   if (getCanonicalType(retType) != LRetType)
     allLTypes = false;
   if (getCanonicalType(retType) != RRetType)
     allRTypes = false;
 
   // FIXME: double check this
   // FIXME: should we error if lbase->getRegParmAttr() != 0 &&
   //                           rbase->getRegParmAttr() != 0 &&
   //                           lbase->getRegParmAttr() != rbase->getRegParmAttr()?
   FunctionType::ExtInfo lbaseInfo = lbase->getExtInfo();
   FunctionType::ExtInfo rbaseInfo = rbase->getExtInfo();
 
   // Compatible functions must have compatible calling conventions
   if (lbaseInfo.getCC() != rbaseInfo.getCC())
     return {};
 
   // Regparm is part of the calling convention.
   if (lbaseInfo.getHasRegParm() != rbaseInfo.getHasRegParm())
     return {};
   if (lbaseInfo.getRegParm() != rbaseInfo.getRegParm())
     return {};
 
   if (lbaseInfo.getProducesResult() != rbaseInfo.getProducesResult())
     return {};
   if (lbaseInfo.getNoCallerSavedRegs() != rbaseInfo.getNoCallerSavedRegs())
     return {};
   if (lbaseInfo.getNoCfCheck() != rbaseInfo.getNoCfCheck())
     return {};
 
   // FIXME: some uses, e.g. conditional exprs, really want this to be 'both'.
   bool NoReturn = lbaseInfo.getNoReturn() || rbaseInfo.getNoReturn();
 
   if (lbaseInfo.getNoReturn() != NoReturn)
     allLTypes = false;
   if (rbaseInfo.getNoReturn() != NoReturn)
     allRTypes = false;
 
   FunctionType::ExtInfo einfo = lbaseInfo.withNoReturn(NoReturn);
 
   if (lproto && rproto) { // two C99 style function prototypes
     assert((AllowCXX ||
             (!lproto->hasExceptionSpec() && !rproto->hasExceptionSpec())) &&
            "C++ shouldn't be here");
     // Compatible functions must have the same number of parameters
     if (lproto->getNumParams() != rproto->getNumParams())
       return {};
 
     // Variadic and non-variadic functions aren't compatible
     if (lproto->isVariadic() != rproto->isVariadic())
       return {};
 
     if (lproto->getMethodQuals() != rproto->getMethodQuals())
       return {};
 
     SmallVector<FunctionProtoType::ExtParameterInfo, 4> newParamInfos;
     bool canUseLeft, canUseRight;
     if (!mergeExtParameterInfo(lproto, rproto, canUseLeft, canUseRight,
                                newParamInfos))
       return {};
 
     if (!canUseLeft)
       allLTypes = false;
     if (!canUseRight)
       allRTypes = false;
 
     // Check parameter type compatibility
     SmallVector<QualType, 10> types;
     for (unsigned i = 0, n = lproto->getNumParams(); i < n; i++) {
       QualType lParamType = lproto->getParamType(i).getUnqualifiedType();
       QualType rParamType = rproto->getParamType(i).getUnqualifiedType();
       QualType paramType = mergeFunctionParameterTypes(
           lParamType, rParamType, OfBlockPointer, Unqualified);
       if (paramType.isNull())
         return {};
 
       if (Unqualified)
         paramType = paramType.getUnqualifiedType();
 
       types.push_back(paramType);
       if (Unqualified) {
         lParamType = lParamType.getUnqualifiedType();
         rParamType = rParamType.getUnqualifiedType();
       }
 
       if (getCanonicalType(paramType) != getCanonicalType(lParamType))
         allLTypes = false;
       if (getCanonicalType(paramType) != getCanonicalType(rParamType))
         allRTypes = false;
     }
 
     if (allLTypes) return lhs;
     if (allRTypes) return rhs;
 
     FunctionProtoType::ExtProtoInfo EPI = lproto->getExtProtoInfo();
     EPI.ExtInfo = einfo;
     EPI.ExtParameterInfos =
         newParamInfos.empty() ? nullptr : newParamInfos.data();
     return getFunctionType(retType, types, EPI);
   }
 
   if (lproto) allRTypes = false;
   if (rproto) allLTypes = false;
 
   const FunctionProtoType *proto = lproto ? lproto : rproto;
   if (proto) {
     assert((AllowCXX || !proto->hasExceptionSpec()) && "C++ shouldn't be here");
     if (proto->isVariadic())
       return {};
     // Check that the types are compatible with the types that
     // would result from default argument promotions (C99 6.7.5.3p15).
     // The only types actually affected are promotable integer
     // types and floats, which would be passed as a different
     // type depending on whether the prototype is visible.
     for (unsigned i = 0, n = proto->getNumParams(); i < n; ++i) {
       QualType paramTy = proto->getParamType(i);
 
       // Look at the converted type of enum types, since that is the type used
       // to pass enum values.
       if (const auto *Enum = paramTy->getAs<EnumType>()) {
         paramTy = Enum->getDecl()->getIntegerType();
         if (paramTy.isNull())
           return {};
       }
 
       if (paramTy->isPromotableIntegerType() ||
           getCanonicalType(paramTy).getUnqualifiedType() == FloatTy)
         return {};
     }
 
     if (allLTypes) return lhs;
     if (allRTypes) return rhs;
 
     FunctionProtoType::ExtProtoInfo EPI = proto->getExtProtoInfo();
     EPI.ExtInfo = einfo;
     return getFunctionType(retType, proto->getParamTypes(), EPI);
   }
 
   if (allLTypes) return lhs;
   if (allRTypes) return rhs;
   return getFunctionNoProtoType(retType, einfo);
 }
 
 /// Given that we have an enum type and a non-enum type, try to merge them.
 static QualType mergeEnumWithInteger(ASTContext &Context, const EnumType *ET,
                                      QualType other, bool isBlockReturnType) {
   // C99 6.7.2.2p4: Each enumerated type shall be compatible with char,
   // a signed integer type, or an unsigned integer type.
   // Compatibility is based on the underlying type, not the promotion
   // type.
   QualType underlyingType = ET->getDecl()->getIntegerType();
   if (underlyingType.isNull())
     return {};
   if (Context.hasSameType(underlyingType, other))
     return other;
 
   // In block return types, we're more permissive and accept any
   // integral type of the same size.
   if (isBlockReturnType && other->isIntegerType() &&
       Context.getTypeSize(underlyingType) == Context.getTypeSize(other))
     return other;
 
   return {};
 }
 
 QualType ASTContext::mergeTypes(QualType LHS, QualType RHS,
                                 bool OfBlockPointer,
                                 bool Unqualified, bool BlockReturnType) {
   // For C++ we will not reach this code with reference types (see below),
   // for OpenMP variant call overloading we might.
   //
   // C++ [expr]: If an expression initially has the type "reference to T", the
   // type is adjusted to "T" prior to any further analysis, the expression
   // designates the object or function denoted by the reference, and the
   // expression is an lvalue unless the reference is an rvalue reference and
   // the expression is a function call (possibly inside parentheses).
   auto *LHSRefTy = LHS->getAs<ReferenceType>();
   auto *RHSRefTy = RHS->getAs<ReferenceType>();
   if (LangOpts.OpenMP && LHSRefTy && RHSRefTy &&
       LHS->getTypeClass() == RHS->getTypeClass())
     return mergeTypes(LHSRefTy->getPointeeType(), RHSRefTy->getPointeeType(),
                       OfBlockPointer, Unqualified, BlockReturnType);
   if (LHSRefTy || RHSRefTy)
     return {};
 
   if (Unqualified) {
     LHS = LHS.getUnqualifiedType();
     RHS = RHS.getUnqualifiedType();
   }
 
   QualType LHSCan = getCanonicalType(LHS),
            RHSCan = getCanonicalType(RHS);
 
   // If two types are identical, they are compatible.
   if (LHSCan == RHSCan)
     return LHS;
 
   // If the qualifiers are different, the types aren't compatible... mostly.
   Qualifiers LQuals = LHSCan.getLocalQualifiers();
   Qualifiers RQuals = RHSCan.getLocalQualifiers();
   if (LQuals != RQuals) {
     // If any of these qualifiers are different, we have a type
     // mismatch.
     if (LQuals.getCVRQualifiers() != RQuals.getCVRQualifiers() ||
         LQuals.getAddressSpace() != RQuals.getAddressSpace() ||
         LQuals.getObjCLifetime() != RQuals.getObjCLifetime() ||
         LQuals.hasUnaligned() != RQuals.hasUnaligned())
       return {};
 
     // Exactly one GC qualifier difference is allowed: __strong is
     // okay if the other type has no GC qualifier but is an Objective
     // C object pointer (i.e. implicitly strong by default).  We fix
     // this by pretending that the unqualified type was actually
     // qualified __strong.
     Qualifiers::GC GC_L = LQuals.getObjCGCAttr();
     Qualifiers::GC GC_R = RQuals.getObjCGCAttr();
     assert((GC_L != GC_R) && "unequal qualifier sets had only equal elements");
 
     if (GC_L == Qualifiers::Weak || GC_R == Qualifiers::Weak)
       return {};
 
     if (GC_L == Qualifiers::Strong && RHSCan->isObjCObjectPointerType()) {
       return mergeTypes(LHS, getObjCGCQualType(RHS, Qualifiers::Strong));
     }
     if (GC_R == Qualifiers::Strong && LHSCan->isObjCObjectPointerType()) {
       return mergeTypes(getObjCGCQualType(LHS, Qualifiers::Strong), RHS);
     }
     return {};
   }
 
   // Okay, qualifiers are equal.
 
   Type::TypeClass LHSClass = LHSCan->getTypeClass();
   Type::TypeClass RHSClass = RHSCan->getTypeClass();
 
   // We want to consider the two function types to be the same for these
   // comparisons, just force one to the other.
   if (LHSClass == Type::FunctionProto) LHSClass = Type::FunctionNoProto;
   if (RHSClass == Type::FunctionProto) RHSClass = Type::FunctionNoProto;
 
   // Same as above for arrays
   if (LHSClass == Type::VariableArray || LHSClass == Type::IncompleteArray)
     LHSClass = Type::ConstantArray;
   if (RHSClass == Type::VariableArray || RHSClass == Type::IncompleteArray)
     RHSClass = Type::ConstantArray;
 
   // ObjCInterfaces are just specialized ObjCObjects.
   if (LHSClass == Type::ObjCInterface) LHSClass = Type::ObjCObject;
   if (RHSClass == Type::ObjCInterface) RHSClass = Type::ObjCObject;
 
   // Canonicalize ExtVector -> Vector.
   if (LHSClass == Type::ExtVector) LHSClass = Type::Vector;
   if (RHSClass == Type::ExtVector) RHSClass = Type::Vector;
 
   // If the canonical type classes don't match.
   if (LHSClass != RHSClass) {
     // Note that we only have special rules for turning block enum
     // returns into block int returns, not vice-versa.
     if (const auto *ETy = LHS->getAs<EnumType>()) {
       return mergeEnumWithInteger(*this, ETy, RHS, false);
     }
     if (const EnumType* ETy = RHS->getAs<EnumType>()) {
       return mergeEnumWithInteger(*this, ETy, LHS, BlockReturnType);
     }
     // allow block pointer type to match an 'id' type.
     if (OfBlockPointer && !BlockReturnType) {
        if (LHS->isObjCIdType() && RHS->isBlockPointerType())
          return LHS;
       if (RHS->isObjCIdType() && LHS->isBlockPointerType())
         return RHS;
     }
 
     return {};
   }
 
   // The canonical type classes match.
   switch (LHSClass) {
 #define TYPE(Class, Base)
 #define ABSTRACT_TYPE(Class, Base)
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
 #define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
 #include "clang/AST/TypeNodes.inc"
     llvm_unreachable("Non-canonical and dependent types shouldn't get here");
 
   case Type::Auto:
   case Type::DeducedTemplateSpecialization:
   case Type::LValueReference:
   case Type::RValueReference:
   case Type::MemberPointer:
     llvm_unreachable("C++ should never be in mergeTypes");
 
   case Type::ObjCInterface:
   case Type::IncompleteArray:
   case Type::VariableArray:
   case Type::FunctionProto:
   case Type::ExtVector:
     llvm_unreachable("Types are eliminated above");
 
   case Type::Pointer:
   {
     // Merge two pointer types, while trying to preserve typedef info
     QualType LHSPointee = LHS->castAs<PointerType>()->getPointeeType();
     QualType RHSPointee = RHS->castAs<PointerType>()->getPointeeType();
     if (Unqualified) {
       LHSPointee = LHSPointee.getUnqualifiedType();
       RHSPointee = RHSPointee.getUnqualifiedType();
     }
     QualType ResultType = mergeTypes(LHSPointee, RHSPointee, false,
                                      Unqualified);
     if (ResultType.isNull())
       return {};
     if (getCanonicalType(LHSPointee) == getCanonicalType(ResultType))
       return LHS;
     if (getCanonicalType(RHSPointee) == getCanonicalType(ResultType))
       return RHS;
     return getPointerType(ResultType);
   }
   case Type::BlockPointer:
   {
     // Merge two block pointer types, while trying to preserve typedef info
     QualType LHSPointee = LHS->castAs<BlockPointerType>()->getPointeeType();
     QualType RHSPointee = RHS->castAs<BlockPointerType>()->getPointeeType();
     if (Unqualified) {
       LHSPointee = LHSPointee.getUnqualifiedType();
       RHSPointee = RHSPointee.getUnqualifiedType();
     }
     if (getLangOpts().OpenCL) {
       Qualifiers LHSPteeQual = LHSPointee.getQualifiers();
       Qualifiers RHSPteeQual = RHSPointee.getQualifiers();
       // Blocks can't be an expression in a ternary operator (OpenCL v2.0
       // 6.12.5) thus the following check is asymmetric.
       if (!LHSPteeQual.isAddressSpaceSupersetOf(RHSPteeQual))
         return {};
       LHSPteeQual.removeAddressSpace();
       RHSPteeQual.removeAddressSpace();
       LHSPointee =
           QualType(LHSPointee.getTypePtr(), LHSPteeQual.getAsOpaqueValue());
       RHSPointee =
           QualType(RHSPointee.getTypePtr(), RHSPteeQual.getAsOpaqueValue());
     }
     QualType ResultType = mergeTypes(LHSPointee, RHSPointee, OfBlockPointer,
                                      Unqualified);
     if (ResultType.isNull())
       return {};
     if (getCanonicalType(LHSPointee) == getCanonicalType(ResultType))
       return LHS;
     if (getCanonicalType(RHSPointee) == getCanonicalType(ResultType))
       return RHS;
     return getBlockPointerType(ResultType);
   }
   case Type::Atomic:
   {
     // Merge two pointer types, while trying to preserve typedef info
     QualType LHSValue = LHS->castAs<AtomicType>()->getValueType();
     QualType RHSValue = RHS->castAs<AtomicType>()->getValueType();
     if (Unqualified) {
       LHSValue = LHSValue.getUnqualifiedType();
       RHSValue = RHSValue.getUnqualifiedType();
     }
     QualType ResultType = mergeTypes(LHSValue, RHSValue, false,
                                      Unqualified);
     if (ResultType.isNull())
       return {};
     if (getCanonicalType(LHSValue) == getCanonicalType(ResultType))
       return LHS;
     if (getCanonicalType(RHSValue) == getCanonicalType(ResultType))
       return RHS;
     return getAtomicType(ResultType);
   }
   case Type::ConstantArray:
   {
     const ConstantArrayType* LCAT = getAsConstantArrayType(LHS);
     const ConstantArrayType* RCAT = getAsConstantArrayType(RHS);
     if (LCAT && RCAT && RCAT->getSize() != LCAT->getSize())
       return {};
 
     QualType LHSElem = getAsArrayType(LHS)->getElementType();
     QualType RHSElem = getAsArrayType(RHS)->getElementType();
     if (Unqualified) {
       LHSElem = LHSElem.getUnqualifiedType();
       RHSElem = RHSElem.getUnqualifiedType();
     }
 
     QualType ResultType = mergeTypes(LHSElem, RHSElem, false, Unqualified);
     if (ResultType.isNull())
       return {};
 
     const VariableArrayType* LVAT = getAsVariableArrayType(LHS);
     const VariableArrayType* RVAT = getAsVariableArrayType(RHS);
 
     // If either side is a variable array, and both are complete, check whether
     // the current dimension is definite.
     if (LVAT || RVAT) {
       auto SizeFetch = [this](const VariableArrayType* VAT,
           const ConstantArrayType* CAT)
           -> std::pair<bool,llvm::APInt> {
         if (VAT) {
           Optional<llvm::APSInt> TheInt;
           Expr *E = VAT->getSizeExpr();
           if (E && (TheInt = E->getIntegerConstantExpr(*this)))
             return std::make_pair(true, *TheInt);
           return std::make_pair(false, llvm::APSInt());
         }
         if (CAT)
           return std::make_pair(true, CAT->getSize());
         return std::make_pair(false, llvm::APInt());
       };
 
       bool HaveLSize, HaveRSize;
       llvm::APInt LSize, RSize;
       std::tie(HaveLSize, LSize) = SizeFetch(LVAT, LCAT);
       std::tie(HaveRSize, RSize) = SizeFetch(RVAT, RCAT);
       if (HaveLSize && HaveRSize && !llvm::APInt::isSameValue(LSize, RSize))
         return {}; // Definite, but unequal, array dimension
     }
 
     if (LCAT && getCanonicalType(LHSElem) == getCanonicalType(ResultType))
       return LHS;
     if (RCAT && getCanonicalType(RHSElem) == getCanonicalType(ResultType))
       return RHS;
     if (LCAT)
       return getConstantArrayType(ResultType, LCAT->getSize(),
                                   LCAT->getSizeExpr(),
                                   ArrayType::ArraySizeModifier(), 0);
     if (RCAT)
       return getConstantArrayType(ResultType, RCAT->getSize(),
                                   RCAT->getSizeExpr(),
                                   ArrayType::ArraySizeModifier(), 0);
     if (LVAT && getCanonicalType(LHSElem) == getCanonicalType(ResultType))
       return LHS;
     if (RVAT && getCanonicalType(RHSElem) == getCanonicalType(ResultType))
       return RHS;
     if (LVAT) {
       // FIXME: This isn't correct! But tricky to implement because
       // the array's size has to be the size of LHS, but the type
       // has to be different.
       return LHS;
     }
     if (RVAT) {
       // FIXME: This isn't correct! But tricky to implement because
       // the array's size has to be the size of RHS, but the type
       // has to be different.
       return RHS;
     }
     if (getCanonicalType(LHSElem) == getCanonicalType(ResultType)) return LHS;
     if (getCanonicalType(RHSElem) == getCanonicalType(ResultType)) return RHS;
     return getIncompleteArrayType(ResultType,
                                   ArrayType::ArraySizeModifier(), 0);
   }
   case Type::FunctionNoProto:
     return mergeFunctionTypes(LHS, RHS, OfBlockPointer, Unqualified);
   case Type::Record:
   case Type::Enum:
     return {};
   case Type::Builtin:
     // Only exactly equal builtin types are compatible, which is tested above.
     return {};
   case Type::Complex:
     // Distinct complex types are incompatible.
     return {};
   case Type::Vector:
     // FIXME: The merged type should be an ExtVector!
     if (areCompatVectorTypes(LHSCan->castAs<VectorType>(),
                              RHSCan->castAs<VectorType>()))
       return LHS;
     return {};
   case Type::ConstantMatrix:
     if (areCompatMatrixTypes(LHSCan->castAs<ConstantMatrixType>(),
                              RHSCan->castAs<ConstantMatrixType>()))
       return LHS;
     return {};
   case Type::ObjCObject: {
     // Check if the types are assignment compatible.
     // FIXME: This should be type compatibility, e.g. whether
     // "LHS x; RHS x;" at global scope is legal.
     if (canAssignObjCInterfaces(LHS->castAs<ObjCObjectType>(),
                                 RHS->castAs<ObjCObjectType>()))
       return LHS;
     return {};
   }
   case Type::ObjCObjectPointer:
     if (OfBlockPointer) {
       if (canAssignObjCInterfacesInBlockPointer(
               LHS->castAs<ObjCObjectPointerType>(),
               RHS->castAs<ObjCObjectPointerType>(), BlockReturnType))
         return LHS;
       return {};
     }
     if (canAssignObjCInterfaces(LHS->castAs<ObjCObjectPointerType>(),
                                 RHS->castAs<ObjCObjectPointerType>()))
       return LHS;
     return {};
   case Type::Pipe:
     assert(LHS != RHS &&
            "Equivalent pipe types should have already been handled!");
     return {};
   case Type::BitInt: {
     // Merge two bit-precise int types, while trying to preserve typedef info.
     bool LHSUnsigned = LHS->castAs<BitIntType>()->isUnsigned();
     bool RHSUnsigned = RHS->castAs<BitIntType>()->isUnsigned();
     unsigned LHSBits = LHS->castAs<BitIntType>()->getNumBits();
     unsigned RHSBits = RHS->castAs<BitIntType>()->getNumBits();
 
     // Like unsigned/int, shouldn't have a type if they don't match.
     if (LHSUnsigned != RHSUnsigned)
       return {};
 
     if (LHSBits != RHSBits)
       return {};
     return LHS;
   }
   }
 
   llvm_unreachable("Invalid Type::Class!");
 }
 
 bool ASTContext::mergeExtParameterInfo(
     const FunctionProtoType *FirstFnType, const FunctionProtoType *SecondFnType,
     bool &CanUseFirst, bool &CanUseSecond,
     SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &NewParamInfos) {
   assert(NewParamInfos.empty() && "param info list not empty");
   CanUseFirst = CanUseSecond = true;
   bool FirstHasInfo = FirstFnType->hasExtParameterInfos();
   bool SecondHasInfo = SecondFnType->hasExtParameterInfos();
 
   // Fast path: if the first type doesn't have ext parameter infos,
   // we match if and only if the second type also doesn't have them.
   if (!FirstHasInfo && !SecondHasInfo)
     return true;
 
   bool NeedParamInfo = false;
   size_t E = FirstHasInfo ? FirstFnType->getExtParameterInfos().size()
                           : SecondFnType->getExtParameterInfos().size();
 
   for (size_t I = 0; I < E; ++I) {
     FunctionProtoType::ExtParameterInfo FirstParam, SecondParam;
     if (FirstHasInfo)
       FirstParam = FirstFnType->getExtParameterInfo(I);
     if (SecondHasInfo)
       SecondParam = SecondFnType->getExtParameterInfo(I);
 
     // Cannot merge unless everything except the noescape flag matches.
     if (FirstParam.withIsNoEscape(false) != SecondParam.withIsNoEscape(false))
       return false;
 
     bool FirstNoEscape = FirstParam.isNoEscape();
     bool SecondNoEscape = SecondParam.isNoEscape();
     bool IsNoEscape = FirstNoEscape && SecondNoEscape;
     NewParamInfos.push_back(FirstParam.withIsNoEscape(IsNoEscape));
     if (NewParamInfos.back().getOpaqueValue())
       NeedParamInfo = true;
     if (FirstNoEscape != IsNoEscape)
       CanUseFirst = false;
     if (SecondNoEscape != IsNoEscape)
       CanUseSecond = false;
   }
 
   if (!NeedParamInfo)
     NewParamInfos.clear();
 
   return true;
 }
 
 void ASTContext::ResetObjCLayout(const ObjCContainerDecl *CD) {
   ObjCLayouts[CD] = nullptr;
 }
 
 /// mergeObjCGCQualifiers - This routine merges ObjC's GC attribute of 'LHS' and
 /// 'RHS' attributes and returns the merged version; including for function
 /// return types.
 QualType ASTContext::mergeObjCGCQualifiers(QualType LHS, QualType RHS) {
   QualType LHSCan = getCanonicalType(LHS),
   RHSCan = getCanonicalType(RHS);
   // If two types are identical, they are compatible.
   if (LHSCan == RHSCan)
     return LHS;
   if (RHSCan->isFunctionType()) {
     if (!LHSCan->isFunctionType())
       return {};
     QualType OldReturnType =
         cast<FunctionType>(RHSCan.getTypePtr())->getReturnType();
     QualType NewReturnType =
         cast<FunctionType>(LHSCan.getTypePtr())->getReturnType();
     QualType ResReturnType =
       mergeObjCGCQualifiers(NewReturnType, OldReturnType);
     if (ResReturnType.isNull())
       return {};
     if (ResReturnType == NewReturnType || ResReturnType == OldReturnType) {
       // id foo(); ... __strong id foo(); or: __strong id foo(); ... id foo();
       // In either case, use OldReturnType to build the new function type.
       const auto *F = LHS->castAs<FunctionType>();
       if (const auto *FPT = cast<FunctionProtoType>(F)) {
         FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo();
         EPI.ExtInfo = getFunctionExtInfo(LHS);
         QualType ResultType =
             getFunctionType(OldReturnType, FPT->getParamTypes(), EPI);
         return ResultType;
       }
     }
     return {};
   }
 
   // If the qualifiers are different, the types can still be merged.
   Qualifiers LQuals = LHSCan.getLocalQualifiers();
   Qualifiers RQuals = RHSCan.getLocalQualifiers();
   if (LQuals != RQuals) {
     // If any of these qualifiers are different, we have a type mismatch.
     if (LQuals.getCVRQualifiers() != RQuals.getCVRQualifiers() ||
         LQuals.getAddressSpace() != RQuals.getAddressSpace())
       return {};
 
     // Exactly one GC qualifier difference is allowed: __strong is
     // okay if the other type has no GC qualifier but is an Objective
     // C object pointer (i.e. implicitly strong by default).  We fix
     // this by pretending that the unqualified type was actually
     // qualified __strong.
     Qualifiers::GC GC_L = LQuals.getObjCGCAttr();
     Qualifiers::GC GC_R = RQuals.getObjCGCAttr();
     assert((GC_L != GC_R) && "unequal qualifier sets had only equal elements");
 
     if (GC_L == Qualifiers::Weak || GC_R == Qualifiers::Weak)
       return {};
 
     if (GC_L == Qualifiers::Strong)
       return LHS;
     if (GC_R == Qualifiers::Strong)
       return RHS;
     return {};
   }
 
   if (LHSCan->isObjCObjectPointerType() && RHSCan->isObjCObjectPointerType()) {
     QualType LHSBaseQT = LHS->castAs<ObjCObjectPointerType>()->getPointeeType();
     QualType RHSBaseQT = RHS->castAs<ObjCObjectPointerType>()->getPointeeType();
     QualType ResQT = mergeObjCGCQualifiers(LHSBaseQT, RHSBaseQT);
     if (ResQT == LHSBaseQT)
       return LHS;
     if (ResQT == RHSBaseQT)
       return RHS;
   }
   return {};
 }
 
 //===----------------------------------------------------------------------===//
 //                         Integer Predicates
 //===----------------------------------------------------------------------===//
 
 unsigned ASTContext::getIntWidth(QualType T) const {
   if (const auto *ET = T->getAs<EnumType>())
     T = ET->getDecl()->getIntegerType();
   if (T->isBooleanType())
     return 1;
   if (const auto *EIT = T->getAs<BitIntType>())
     return EIT->getNumBits();
   // For builtin types, just use the standard type sizing method
   return (unsigned)getTypeSize(T);
 }
 
 QualType ASTContext::getCorrespondingUnsignedType(QualType T) const {
   assert((T->hasSignedIntegerRepresentation() || T->isSignedFixedPointType()) &&
          "Unexpected type");
 
   // Turn <4 x signed int> -> <4 x unsigned int>
   if (const auto *VTy = T->getAs<VectorType>())
     return getVectorType(getCorrespondingUnsignedType(VTy->getElementType()),
                          VTy->getNumElements(), VTy->getVectorKind());
 
   // For _BitInt, return an unsigned _BitInt with same width.
   if (const auto *EITy = T->getAs<BitIntType>())
     return getBitIntType(/*Unsigned=*/true, EITy->getNumBits());
 
   // For enums, get the underlying integer type of the enum, and let the general
   // integer type signchanging code handle it.
   if (const auto *ETy = T->getAs<EnumType>())
     T = ETy->getDecl()->getIntegerType();
 
   switch (T->castAs<BuiltinType>()->getKind()) {
   case BuiltinType::Char_S:
   case BuiltinType::SChar:
     return UnsignedCharTy;
   case BuiltinType::Short:
     return UnsignedShortTy;
   case BuiltinType::Int:
     return UnsignedIntTy;
   case BuiltinType::Long:
     return UnsignedLongTy;
   case BuiltinType::LongLong:
     return UnsignedLongLongTy;
   case BuiltinType::Int128:
     return UnsignedInt128Ty;
   // wchar_t is special. It is either signed or not, but when it's signed,
   // there's no matching "unsigned wchar_t". Therefore we return the unsigned
   // version of it's underlying type instead.
   case BuiltinType::WChar_S:
     return getUnsignedWCharType();
 
   case BuiltinType::ShortAccum:
     return UnsignedShortAccumTy;
   case BuiltinType::Accum:
     return UnsignedAccumTy;
   case BuiltinType::LongAccum:
     return UnsignedLongAccumTy;
   case BuiltinType::SatShortAccum:
     return SatUnsignedShortAccumTy;
   case BuiltinType::SatAccum:
     return SatUnsignedAccumTy;
   case BuiltinType::SatLongAccum:
     return SatUnsignedLongAccumTy;
   case BuiltinType::ShortFract:
     return UnsignedShortFractTy;
   case BuiltinType::Fract:
     return UnsignedFractTy;
   case BuiltinType::LongFract:
     return UnsignedLongFractTy;
   case BuiltinType::SatShortFract:
     return SatUnsignedShortFractTy;
   case BuiltinType::SatFract:
     return SatUnsignedFractTy;
   case BuiltinType::SatLongFract:
     return SatUnsignedLongFractTy;
   default:
     llvm_unreachable("Unexpected signed integer or fixed point type");
   }
 }
 
 QualType ASTContext::getCorrespondingSignedType(QualType T) const {
   assert((T->hasUnsignedIntegerRepresentation() ||
           T->isUnsignedFixedPointType()) &&
          "Unexpected type");
 
   // Turn <4 x unsigned int> -> <4 x signed int>
   if (const auto *VTy = T->getAs<VectorType>())
     return getVectorType(getCorrespondingSignedType(VTy->getElementType()),
                          VTy->getNumElements(), VTy->getVectorKind());
 
   // For _BitInt, return a signed _BitInt with same width.
   if (const auto *EITy = T->getAs<BitIntType>())
     return getBitIntType(/*Unsigned=*/false, EITy->getNumBits());
 
   // For enums, get the underlying integer type of the enum, and let the general
   // integer type signchanging code handle it.
   if (const auto *ETy = T->getAs<EnumType>())
     T = ETy->getDecl()->getIntegerType();
 
   switch (T->castAs<BuiltinType>()->getKind()) {
   case BuiltinType::Char_U:
   case BuiltinType::UChar:
     return SignedCharTy;
   case BuiltinType::UShort:
     return ShortTy;
   case BuiltinType::UInt:
     return IntTy;
   case BuiltinType::ULong:
     return LongTy;
   case BuiltinType::ULongLong:
     return LongLongTy;
   case BuiltinType::UInt128:
     return Int128Ty;
   // wchar_t is special. It is either unsigned or not, but when it's unsigned,
   // there's no matching "signed wchar_t". Therefore we return the signed
   // version of it's underlying type instead.
   case BuiltinType::WChar_U:
     return getSignedWCharType();
 
   case BuiltinType::UShortAccum:
     return ShortAccumTy;
   case BuiltinType::UAccum:
     return AccumTy;
   case BuiltinType::ULongAccum:
     return LongAccumTy;
   case BuiltinType::SatUShortAccum:
     return SatShortAccumTy;
   case BuiltinType::SatUAccum:
     return SatAccumTy;
   case BuiltinType::SatULongAccum:
     return SatLongAccumTy;
   case BuiltinType::UShortFract:
     return ShortFractTy;
   case BuiltinType::UFract:
     return FractTy;
   case BuiltinType::ULongFract:
     return LongFractTy;
   case BuiltinType::SatUShortFract:
     return SatShortFractTy;
   case BuiltinType::SatUFract:
     return SatFractTy;
   case BuiltinType::SatULongFract:
     return SatLongFractTy;
   default:
     llvm_unreachable("Unexpected unsigned integer or fixed point type");
   }
 }
 
 ASTMutationListener::~ASTMutationListener() = default;
 
 void ASTMutationListener::DeducedReturnType(const FunctionDecl *FD,
                                             QualType ReturnType) {}
 
 //===----------------------------------------------------------------------===//
 //                          Builtin Type Computation
 //===----------------------------------------------------------------------===//
 
 /// DecodeTypeFromStr - This decodes one type descriptor from Str, advancing the
 /// pointer over the consumed characters.  This returns the resultant type.  If
 /// AllowTypeModifiers is false then modifier like * are not parsed, just basic
 /// types.  This allows "v2i*" to be parsed as a pointer to a v2i instead of
 /// a vector of "i*".
 ///
 /// RequiresICE is filled in on return to indicate whether the value is required
 /// to be an Integer Constant Expression.
 static QualType DecodeTypeFromStr(const char *&Str, const ASTContext &Context,
                                   ASTContext::GetBuiltinTypeError &Error,
                                   bool &RequiresICE,
                                   bool AllowTypeModifiers) {
   // Modifiers.
   int HowLong = 0;
   bool Signed = false, Unsigned = false;
   RequiresICE = false;
 
   // Read the prefixed modifiers first.
   bool Done = false;
   #ifndef NDEBUG
   bool IsSpecial = false;
   #endif
   while (!Done) {
     switch (*Str++) {
     default: Done = true; --Str; break;
     case 'I':
       RequiresICE = true;
       break;
     case 'S':
       assert(!Unsigned && "Can't use both 'S' and 'U' modifiers!");
       assert(!Signed && "Can't use 'S' modifier multiple times!");
       Signed = true;
       break;
     case 'U':
       assert(!Signed && "Can't use both 'S' and 'U' modifiers!");
       assert(!Unsigned && "Can't use 'U' modifier multiple times!");
       Unsigned = true;
       break;
     case 'L':
       assert(!IsSpecial && "Can't use 'L' with 'W', 'N', 'Z' or 'O' modifiers");
       assert(HowLong <= 2 && "Can't have LLLL modifier");
       ++HowLong;
       break;
     case 'N':
       // 'N' behaves like 'L' for all non LP64 targets and 'int' otherwise.
       assert(!IsSpecial && "Can't use two 'N', 'W', 'Z' or 'O' modifiers!");
       assert(HowLong == 0 && "Can't use both 'L' and 'N' modifiers!");
       #ifndef NDEBUG
       IsSpecial = true;
       #endif
       if (Context.getTargetInfo().getLongWidth() == 32)
         ++HowLong;
       break;
     case 'W':
       // This modifier represents int64 type.
       assert(!IsSpecial && "Can't use two 'N', 'W', 'Z' or 'O' modifiers!");
       assert(HowLong == 0 && "Can't use both 'L' and 'W' modifiers!");
       #ifndef NDEBUG
       IsSpecial = true;
       #endif
       switch (Context.getTargetInfo().getInt64Type()) {
       default:
         llvm_unreachable("Unexpected integer type");
       case TargetInfo::SignedLong:
         HowLong = 1;
         break;
       case TargetInfo::SignedLongLong:
         HowLong = 2;
         break;
       }
       break;
     case 'Z':
       // This modifier represents int32 type.
       assert(!IsSpecial && "Can't use two 'N', 'W', 'Z' or 'O' modifiers!");
       assert(HowLong == 0 && "Can't use both 'L' and 'Z' modifiers!");
       #ifndef NDEBUG
       IsSpecial = true;
       #endif
       switch (Context.getTargetInfo().getIntTypeByWidth(32, true)) {
       default:
         llvm_unreachable("Unexpected integer type");
       case TargetInfo::SignedInt:
         HowLong = 0;
         break;
       case TargetInfo::SignedLong:
         HowLong = 1;
         break;
       case TargetInfo::SignedLongLong:
         HowLong = 2;
         break;
       }
       break;
     case 'O':
       assert(!IsSpecial && "Can't use two 'N', 'W', 'Z' or 'O' modifiers!");
       assert(HowLong == 0 && "Can't use both 'L' and 'O' modifiers!");
       #ifndef NDEBUG
       IsSpecial = true;
       #endif
       if (Context.getLangOpts().OpenCL)
         HowLong = 1;
       else
         HowLong = 2;
       break;
     }
   }
 
   QualType Type;
 
   // Read the base type.
   switch (*Str++) {
   default: llvm_unreachable("Unknown builtin type letter!");
   case 'x':
     assert(HowLong == 0 && !Signed && !Unsigned &&
            "Bad modifiers used with 'x'!");
     Type = Context.Float16Ty;
     break;
   case 'y':
     assert(HowLong == 0 && !Signed && !Unsigned &&
            "Bad modifiers used with 'y'!");
     Type = Context.BFloat16Ty;
     break;
   case 'v':
     assert(HowLong == 0 && !Signed && !Unsigned &&
            "Bad modifiers used with 'v'!");
     Type = Context.VoidTy;
     break;
   case 'h':
     assert(HowLong == 0 && !Signed && !Unsigned &&
            "Bad modifiers used with 'h'!");
     Type = Context.HalfTy;
     break;
   case 'f':
     assert(HowLong == 0 && !Signed && !Unsigned &&
            "Bad modifiers used with 'f'!");
     Type = Context.FloatTy;
     break;
   case 'd':
     assert(HowLong < 3 && !Signed && !Unsigned &&
            "Bad modifiers used with 'd'!");
     if (HowLong == 1)
       Type = Context.LongDoubleTy;
     else if (HowLong == 2)
       Type = Context.Float128Ty;
     else
       Type = Context.DoubleTy;
     break;
   case 's':
     assert(HowLong == 0 && "Bad modifiers used with 's'!");
     if (Unsigned)
       Type = Context.UnsignedShortTy;
     else
       Type = Context.ShortTy;
     break;
   case 'i':
     if (HowLong == 3)
       Type = Unsigned ? Context.UnsignedInt128Ty : Context.Int128Ty;
     else if (HowLong == 2)
       Type = Unsigned ? Context.UnsignedLongLongTy : Context.LongLongTy;
     else if (HowLong == 1)
       Type = Unsigned ? Context.UnsignedLongTy : Context.LongTy;
     else
       Type = Unsigned ? Context.UnsignedIntTy : Context.IntTy;
     break;
   case 'c':
     assert(HowLong == 0 && "Bad modifiers used with 'c'!");
     if (Signed)
       Type = Context.SignedCharTy;
     else if (Unsigned)
       Type = Context.UnsignedCharTy;
     else
       Type = Context.CharTy;
     break;
   case 'b': // boolean
     assert(HowLong == 0 && !Signed && !Unsigned && "Bad modifiers for 'b'!");
     Type = Context.BoolTy;
     break;
   case 'z':  // size_t.
     assert(HowLong == 0 && !Signed && !Unsigned && "Bad modifiers for 'z'!");
     Type = Context.getSizeType();
     break;
   case 'w':  // wchar_t.
     assert(HowLong == 0 && !Signed && !Unsigned && "Bad modifiers for 'w'!");
     Type = Context.getWideCharType();
     break;
   case 'F':
     Type = Context.getCFConstantStringType();
     break;
   case 'G':
     Type = Context.getObjCIdType();
     break;
   case 'H':
     Type = Context.getObjCSelType();
     break;
   case 'M':
     Type = Context.getObjCSuperType();
     break;
   case 'a':
     Type = Context.getBuiltinVaListType();
     assert(!Type.isNull() && "builtin va list type not initialized!");
     break;
   case 'A':
     // This is a "reference" to a va_list; however, what exactly
     // this means depends on how va_list is defined. There are two
     // different kinds of va_list: ones passed by value, and ones
     // passed by reference.  An example of a by-value va_list is
     // x86, where va_list is a char*. An example of by-ref va_list
     // is x86-64, where va_list is a __va_list_tag[1]. For x86,
     // we want this argument to be a char*&; for x86-64, we want
     // it to be a __va_list_tag*.
     Type = Context.getBuiltinVaListType();
     assert(!Type.isNull() && "builtin va list type not initialized!");
     if (Type->isArrayType())
       Type = Context.getArrayDecayedType(Type);
     else
       Type = Context.getLValueReferenceType(Type);
     break;
   case 'q': {
     char *End;
     unsigned NumElements = strtoul(Str, &End, 10);
     assert(End != Str && "Missing vector size");
     Str = End;
 
     QualType ElementType = DecodeTypeFromStr(Str, Context, Error,
                                              RequiresICE, false);
     assert(!RequiresICE && "Can't require vector ICE");
 
     Type = Context.getScalableVectorType(ElementType, NumElements);
     break;
   }
   case 'V': {
     char *End;
     unsigned NumElements = strtoul(Str, &End, 10);
     assert(End != Str && "Missing vector size");
     Str = End;
 
     QualType ElementType = DecodeTypeFromStr(Str, Context, Error,
                                              RequiresICE, false);
     assert(!RequiresICE && "Can't require vector ICE");
 
     // TODO: No way to make AltiVec vectors in builtins yet.
     Type = Context.getVectorType(ElementType, NumElements,
                                  VectorType::GenericVector);
     break;
   }
   case 'E': {
     char *End;
 
     unsigned NumElements = strtoul(Str, &End, 10);
     assert(End != Str && "Missing vector size");
 
     Str = End;
 
     QualType ElementType = DecodeTypeFromStr(Str, Context, Error, RequiresICE,
                                              false);
     Type = Context.getExtVectorType(ElementType, NumElements);
     break;
   }
   case 'X': {
     QualType ElementType = DecodeTypeFromStr(Str, Context, Error, RequiresICE,
                                              false);
     assert(!RequiresICE && "Can't require complex ICE");
     Type = Context.getComplexType(ElementType);
     break;
   }
   case 'Y':
     Type = Context.getPointerDiffType();
     break;
   case 'P':
     Type = Context.getFILEType();
     if (Type.isNull()) {
       Error = ASTContext::GE_Missing_stdio;
       return {};
     }
     break;
   case 'J':
     if (Signed)
       Type = Context.getsigjmp_bufType();
     else
       Type = Context.getjmp_bufType();
 
     if (Type.isNull()) {
       Error = ASTContext::GE_Missing_setjmp;
       return {};
     }
     break;
   case 'K':
     assert(HowLong == 0 && !Signed && !Unsigned && "Bad modifiers for 'K'!");
     Type = Context.getucontext_tType();
 
     if (Type.isNull()) {
       Error = ASTContext::GE_Missing_ucontext;
       return {};
     }
     break;
   case 'p':
     Type = Context.getProcessIDType();
     break;
   }
 
   // If there are modifiers and if we're allowed to parse them, go for it.
   Done = !AllowTypeModifiers;
   while (!Done) {
     switch (char c = *Str++) {
     default: Done = true; --Str; break;
     case '*':
     case '&': {
       // Both pointers and references can have their pointee types
       // qualified with an address space.
       char *End;
       unsigned AddrSpace = strtoul(Str, &End, 10);
       if (End != Str) {
         // Note AddrSpace == 0 is not the same as an unspecified address space.
         Type = Context.getAddrSpaceQualType(
           Type,
           Context.getLangASForBuiltinAddressSpace(AddrSpace));
         Str = End;
       }
       if (c == '*')
         Type = Context.getPointerType(Type);
       else
         Type = Context.getLValueReferenceType(Type);
       break;
     }
     // FIXME: There's no way to have a built-in with an rvalue ref arg.
     case 'C':
       Type = Type.withConst();
       break;
     case 'D':
       Type = Context.getVolatileType(Type);
       break;
     case 'R':
       Type = Type.withRestrict();
       break;
     }
   }
 
   assert((!RequiresICE || Type->isIntegralOrEnumerationType()) &&
          "Integer constant 'I' type must be an integer");
 
   return Type;
 }
 
 // On some targets such as PowerPC, some of the builtins are defined with custom
 // type descriptors for target-dependent types. These descriptors are decoded in
 // other functions, but it may be useful to be able to fall back to default
 // descriptor decoding to define builtins mixing target-dependent and target-
 // independent types. This function allows decoding one type descriptor with
 // default decoding.
 QualType ASTContext::DecodeTypeStr(const char *&Str, const ASTContext &Context,
                                    GetBuiltinTypeError &Error, bool &RequireICE,
                                    bool AllowTypeModifiers) const {
   return DecodeTypeFromStr(Str, Context, Error, RequireICE, AllowTypeModifiers);
 }
 
 /// GetBuiltinType - Return the type for the specified builtin.
 QualType ASTContext::GetBuiltinType(unsigned Id,
                                     GetBuiltinTypeError &Error,
                                     unsigned *IntegerConstantArgs) const {
   const char *TypeStr = BuiltinInfo.getTypeString(Id);
   if (TypeStr[0] == '\0') {
     Error = GE_Missing_type;
     return {};
   }
 
   SmallVector<QualType, 8> ArgTypes;
 
   bool RequiresICE = false;
   Error = GE_None;
   QualType ResType = DecodeTypeFromStr(TypeStr, *this, Error,
                                        RequiresICE, true);
   if (Error != GE_None)
     return {};
 
   assert(!RequiresICE && "Result of intrinsic cannot be required to be an ICE");
 
   while (TypeStr[0] && TypeStr[0] != '.') {
     QualType Ty = DecodeTypeFromStr(TypeStr, *this, Error, RequiresICE, true);
     if (Error != GE_None)
       return {};
 
     // If this argument is required to be an IntegerConstantExpression and the
     // caller cares, fill in the bitmask we return.
     if (RequiresICE && IntegerConstantArgs)
       *IntegerConstantArgs |= 1 << ArgTypes.size();
 
     // Do array -> pointer decay.  The builtin should use the decayed type.
     if (Ty->isArrayType())
       Ty = getArrayDecayedType(Ty);
 
     ArgTypes.push_back(Ty);
   }
 
   if (Id == Builtin::BI__GetExceptionInfo)
     return {};
 
   assert((TypeStr[0] != '.' || TypeStr[1] == 0) &&
          "'.' should only occur at end of builtin type list!");
 
   bool Variadic = (TypeStr[0] == '.');
 
   FunctionType::ExtInfo EI(getDefaultCallingConvention(
       Variadic, /*IsCXXMethod=*/false, /*IsBuiltin=*/true));
   if (BuiltinInfo.isNoReturn(Id)) EI = EI.withNoReturn(true);
 
 
   // We really shouldn't be making a no-proto type here.
   if (ArgTypes.empty() && Variadic && !getLangOpts().CPlusPlus)
     return getFunctionNoProtoType(ResType, EI);
 
   FunctionProtoType::ExtProtoInfo EPI;
   EPI.ExtInfo = EI;
   EPI.Variadic = Variadic;
   if (getLangOpts().CPlusPlus && BuiltinInfo.isNoThrow(Id))
     EPI.ExceptionSpec.Type =
         getLangOpts().CPlusPlus11 ? EST_BasicNoexcept : EST_DynamicNone;
 
   return getFunctionType(ResType, ArgTypes, EPI);
 }
 
 static GVALinkage basicGVALinkageForFunction(const ASTContext &Context,
                                              const FunctionDecl *FD) {
   if (!FD->isExternallyVisible())
     return GVA_Internal;
 
   // Non-user-provided functions get emitted as weak definitions with every
   // use, no matter whether they've been explicitly instantiated etc.
   if (const auto *MD = dyn_cast<CXXMethodDecl>(FD))
     if (!MD->isUserProvided())
       return GVA_DiscardableODR;
 
   GVALinkage External;
   switch (FD->getTemplateSpecializationKind()) {
   case TSK_Undeclared:
   case TSK_ExplicitSpecialization:
     External = GVA_StrongExternal;
     break;
 
   case TSK_ExplicitInstantiationDefinition:
     return GVA_StrongODR;
 
   // C++11 [temp.explicit]p10:
   //   [ Note: The intent is that an inline function that is the subject of
   //   an explicit instantiation declaration will still be implicitly
   //   instantiated when used so that the body can be considered for
   //   inlining, but that no out-of-line copy of the inline function would be
   //   generated in the translation unit. -- end note ]
   case TSK_ExplicitInstantiationDeclaration:
     return GVA_AvailableExternally;
 
   case TSK_ImplicitInstantiation:
     External = GVA_DiscardableODR;
     break;
   }
 
   if (!FD->isInlined())
     return External;
 
   if ((!Context.getLangOpts().CPlusPlus &&
        !Context.getTargetInfo().getCXXABI().isMicrosoft() &&
        !FD->hasAttr<DLLExportAttr>()) ||
       FD->hasAttr<GNUInlineAttr>()) {
     // FIXME: This doesn't match gcc's behavior for dllexport inline functions.
 
     // GNU or C99 inline semantics. Determine whether this symbol should be
     // externally visible.
     if (FD->isInlineDefinitionExternallyVisible())
       return External;
 
     // C99 inline semantics, where the symbol is not externally visible.
     return GVA_AvailableExternally;
   }
 
   // Functions specified with extern and inline in -fms-compatibility mode
   // forcibly get emitted.  While the body of the function cannot be later
   // replaced, the function definition cannot be discarded.
   if (FD->isMSExternInline())
     return GVA_StrongODR;
 
   return GVA_DiscardableODR;
 }
 
 static GVALinkage adjustGVALinkageForAttributes(const ASTContext &Context,
                                                 const Decl *D, GVALinkage L) {
   // See http://msdn.microsoft.com/en-us/library/xa0d9ste.aspx
   // dllexport/dllimport on inline functions.
   if (D->hasAttr<DLLImportAttr>()) {
     if (L == GVA_DiscardableODR || L == GVA_StrongODR)
       return GVA_AvailableExternally;
   } else if (D->hasAttr<DLLExportAttr>()) {
     if (L == GVA_DiscardableODR)
       return GVA_StrongODR;
   } else if (Context.getLangOpts().CUDA && Context.getLangOpts().CUDAIsDevice) {
     // Device-side functions with __global__ attribute must always be
     // visible externally so they can be launched from host.
     if (D->hasAttr<CUDAGlobalAttr>() &&
         (L == GVA_DiscardableODR || L == GVA_Internal))
       return GVA_StrongODR;
     // Single source offloading languages like CUDA/HIP need to be able to
     // access static device variables from host code of the same compilation
     // unit. This is done by externalizing the static variable with a shared
     // name between the host and device compilation which is the same for the
     // same compilation unit whereas different among different compilation
     // units.
     if (Context.shouldExternalizeStaticVar(D))
       return GVA_StrongExternal;
   }
   return L;
 }
 
 /// Adjust the GVALinkage for a declaration based on what an external AST source
 /// knows about whether there can be other definitions of this declaration.
 static GVALinkage
 adjustGVALinkageForExternalDefinitionKind(const ASTContext &Ctx, const Decl *D,
                                           GVALinkage L) {
   ExternalASTSource *Source = Ctx.getExternalSource();
   if (!Source)
     return L;
 
   switch (Source->hasExternalDefinitions(D)) {
   case ExternalASTSource::EK_Never:
     // Other translation units rely on us to provide the definition.
     if (L == GVA_DiscardableODR)
       return GVA_StrongODR;
     break;
 
   case ExternalASTSource::EK_Always:
     return GVA_AvailableExternally;
 
   case ExternalASTSource::EK_ReplyHazy:
     break;
   }
   return L;
 }
 
 GVALinkage ASTContext::GetGVALinkageForFunction(const FunctionDecl *FD) const {
   return adjustGVALinkageForExternalDefinitionKind(*this, FD,
            adjustGVALinkageForAttributes(*this, FD,
              basicGVALinkageForFunction(*this, FD)));
 }
 
 static GVALinkage basicGVALinkageForVariable(const ASTContext &Context,
                                              const VarDecl *VD) {
   if (!VD->isExternallyVisible())
     return GVA_Internal;
 
   if (VD->isStaticLocal()) {
     const DeclContext *LexicalContext = VD->getParentFunctionOrMethod();
     while (LexicalContext && !isa<FunctionDecl>(LexicalContext))
       LexicalContext = LexicalContext->getLexicalParent();
 
     // ObjC Blocks can create local variables that don't have a FunctionDecl
     // LexicalContext.
     if (!LexicalContext)
       return GVA_DiscardableODR;
 
     // Otherwise, let the static local variable inherit its linkage from the
     // nearest enclosing function.
     auto StaticLocalLinkage =
         Context.GetGVALinkageForFunction(cast<FunctionDecl>(LexicalContext));
 
     // Itanium ABI 5.2.2: "Each COMDAT group [for a static local variable] must
     // be emitted in any object with references to the symbol for the object it
     // contains, whether inline or out-of-line."
     // Similar behavior is observed with MSVC. An alternative ABI could use
     // StrongODR/AvailableExternally to match the function, but none are
     // known/supported currently.
     if (StaticLocalLinkage == GVA_StrongODR ||
         StaticLocalLinkage == GVA_AvailableExternally)
       return GVA_DiscardableODR;
     return StaticLocalLinkage;
   }
 
   // MSVC treats in-class initialized static data members as definitions.
   // By giving them non-strong linkage, out-of-line definitions won't
   // cause link errors.
   if (Context.isMSStaticDataMemberInlineDefinition(VD))
     return GVA_DiscardableODR;
 
   // Most non-template variables have strong linkage; inline variables are
   // linkonce_odr or (occasionally, for compatibility) weak_odr.
   GVALinkage StrongLinkage;
   switch (Context.getInlineVariableDefinitionKind(VD)) {
   case ASTContext::InlineVariableDefinitionKind::None:
     StrongLinkage = GVA_StrongExternal;
     break;
   case ASTContext::InlineVariableDefinitionKind::Weak:
   case ASTContext::InlineVariableDefinitionKind::WeakUnknown:
     StrongLinkage = GVA_DiscardableODR;
     break;
   case ASTContext::InlineVariableDefinitionKind::Strong:
     StrongLinkage = GVA_StrongODR;
     break;
   }
 
   switch (VD->getTemplateSpecializationKind()) {
   case TSK_Undeclared:
     return StrongLinkage;
 
   case TSK_ExplicitSpecialization:
     return Context.getTargetInfo().getCXXABI().isMicrosoft() &&
                    VD->isStaticDataMember()
                ? GVA_StrongODR
                : StrongLinkage;
 
   case TSK_ExplicitInstantiationDefinition:
     return GVA_StrongODR;
 
   case TSK_ExplicitInstantiationDeclaration:
     return GVA_AvailableExternally;
 
   case TSK_ImplicitInstantiation:
     return GVA_DiscardableODR;
   }
 
   llvm_unreachable("Invalid Linkage!");
 }
 
 GVALinkage ASTContext::GetGVALinkageForVariable(const VarDecl *VD) {
   return adjustGVALinkageForExternalDefinitionKind(*this, VD,
            adjustGVALinkageForAttributes(*this, VD,
              basicGVALinkageForVariable(*this, VD)));
 }
 
 bool ASTContext::DeclMustBeEmitted(const Decl *D) {
   if (const auto *VD = dyn_cast<VarDecl>(D)) {
     if (!VD->isFileVarDecl())
       return false;
     // Global named register variables (GNU extension) are never emitted.
     if (VD->getStorageClass() == SC_Register)
       return false;
     if (VD->getDescribedVarTemplate() ||
         isa<VarTemplatePartialSpecializationDecl>(VD))
       return false;
   } else if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
     // We never need to emit an uninstantiated function template.
     if (FD->getTemplatedKind() == FunctionDecl::TK_FunctionTemplate)
       return false;
   } else if (isa<PragmaCommentDecl>(D))
     return true;
   else if (isa<PragmaDetectMismatchDecl>(D))
     return true;
   else if (isa<OMPRequiresDecl>(D))
     return true;
   else if (isa<OMPThreadPrivateDecl>(D))
     return !D->getDeclContext()->isDependentContext();
   else if (isa<OMPAllocateDecl>(D))
     return !D->getDeclContext()->isDependentContext();
   else if (isa<OMPDeclareReductionDecl>(D) || isa<OMPDeclareMapperDecl>(D))
     return !D->getDeclContext()->isDependentContext();
   else if (isa<ImportDecl>(D))
     return true;
   else
     return false;
 
   // If this is a member of a class template, we do not need to emit it.
   if (D->getDeclContext()->isDependentContext())
     return false;
 
   // Weak references don't produce any output by themselves.
   if (D->hasAttr<WeakRefAttr>())
     return false;
 
   // Aliases and used decls are required.
   if (D->hasAttr<AliasAttr>() || D->hasAttr<UsedAttr>())
     return true;
 
   if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
     // Forward declarations aren't required.
     if (!FD->doesThisDeclarationHaveABody())
       return FD->doesDeclarationForceExternallyVisibleDefinition();
 
     // Constructors and destructors are required.
     if (FD->hasAttr<ConstructorAttr>() || FD->hasAttr<DestructorAttr>())
       return true;
 
     // The key function for a class is required.  This rule only comes
     // into play when inline functions can be key functions, though.
     if (getTargetInfo().getCXXABI().canKeyFunctionBeInline()) {
       if (const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
         const CXXRecordDecl *RD = MD->getParent();
         if (MD->isOutOfLine() && RD->isDynamicClass()) {
           const CXXMethodDecl *KeyFunc = getCurrentKeyFunction(RD);
           if (KeyFunc && KeyFunc->getCanonicalDecl() == MD->getCanonicalDecl())
             return true;
         }
       }
     }
 
     GVALinkage Linkage = GetGVALinkageForFunction(FD);
 
     // static, static inline, always_inline, and extern inline functions can
     // always be deferred.  Normal inline functions can be deferred in C99/C++.
     // Implicit template instantiations can also be deferred in C++.
     return !isDiscardableGVALinkage(Linkage);
   }
 
   const auto *VD = cast<VarDecl>(D);
   assert(VD->isFileVarDecl() && "Expected file scoped var");
 
   // If the decl is marked as `declare target to`, it should be emitted for the
   // host and for the device.
   if (LangOpts.OpenMP &&
       OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD))
     return true;
 
   if (VD->isThisDeclarationADefinition() == VarDecl::DeclarationOnly &&
       !isMSStaticDataMemberInlineDefinition(VD))
     return false;
 
   // Variables that can be needed in other TUs are required.
   auto Linkage = GetGVALinkageForVariable(VD);
   if (!isDiscardableGVALinkage(Linkage))
     return true;
 
   // We never need to emit a variable that is available in another TU.
   if (Linkage == GVA_AvailableExternally)
     return false;
 
   // Variables that have destruction with side-effects are required.
   if (VD->needsDestruction(*this))
     return true;
 
   // Variables that have initialization with side-effects are required.
   if (VD->getInit() && VD->getInit()->HasSideEffects(*this) &&
       // We can get a value-dependent initializer during error recovery.
       (VD->getInit()->isValueDependent() || !VD->evaluateValue()))
     return true;
 
   // Likewise, variables with tuple-like bindings are required if their
   // bindings have side-effects.
   if (const auto *DD = dyn_cast<DecompositionDecl>(VD))
     for (const auto *BD : DD->bindings())
       if (const auto *BindingVD = BD->getHoldingVar())
         if (DeclMustBeEmitted(BindingVD))
           return true;
 
   return false;
 }
 
 void ASTContext::forEachMultiversionedFunctionVersion(
     const FunctionDecl *FD,
     llvm::function_ref<void(FunctionDecl *)> Pred) const {
   assert(FD->isMultiVersion() && "Only valid for multiversioned functions");
   llvm::SmallDenseSet<const FunctionDecl*, 4> SeenDecls;
   FD = FD->getMostRecentDecl();
   // FIXME: The order of traversal here matters and depends on the order of
   // lookup results, which happens to be (mostly) oldest-to-newest, but we
   // shouldn't rely on that.
   for (auto *CurDecl :
        FD->getDeclContext()->getRedeclContext()->lookup(FD->getDeclName())) {
     FunctionDecl *CurFD = CurDecl->getAsFunction()->getMostRecentDecl();
     if (CurFD && hasSameType(CurFD->getType(), FD->getType()) &&
         std::end(SeenDecls) == llvm::find(SeenDecls, CurFD)) {
       SeenDecls.insert(CurFD);
       Pred(CurFD);
     }
   }
 }
 
 CallingConv ASTContext::getDefaultCallingConvention(bool IsVariadic,
                                                     bool IsCXXMethod,
                                                     bool IsBuiltin) const {
   // Pass through to the C++ ABI object
   if (IsCXXMethod)
     return ABI->getDefaultMethodCallConv(IsVariadic);
 
   // Builtins ignore user-specified default calling convention and remain the
   // Target's default calling convention.
   if (!IsBuiltin) {
     switch (LangOpts.getDefaultCallingConv()) {
     case LangOptions::DCC_None:
       break;
     case LangOptions::DCC_CDecl:
       return CC_C;
     case LangOptions::DCC_FastCall:
       if (getTargetInfo().hasFeature("sse2") && !IsVariadic)
         return CC_X86FastCall;
       break;
     case LangOptions::DCC_StdCall:
       if (!IsVariadic)
         return CC_X86StdCall;
       break;
     case LangOptions::DCC_VectorCall:
       // __vectorcall cannot be applied to variadic functions.
       if (!IsVariadic)
         return CC_X86VectorCall;
       break;
     case LangOptions::DCC_RegCall:
       // __regcall cannot be applied to variadic functions.
       if (!IsVariadic)
         return CC_X86RegCall;
       break;
     }
   }
   return Target->getDefaultCallingConv();
 }
 
 bool ASTContext::isNearlyEmpty(const CXXRecordDecl *RD) const {
   // Pass through to the C++ ABI object
   return ABI->isNearlyEmpty(RD);
 }
 
 VTableContextBase *ASTContext::getVTableContext() {
   if (!VTContext.get()) {
     auto ABI = Target->getCXXABI();
     if (ABI.isMicrosoft())
       VTContext.reset(new MicrosoftVTableContext(*this));
     else {
       auto ComponentLayout = getLangOpts().RelativeCXXABIVTables
                                  ? ItaniumVTableContext::Relative
                                  : ItaniumVTableContext::Pointer;
       VTContext.reset(new ItaniumVTableContext(*this, ComponentLayout));
     }
   }
   return VTContext.get();
 }
 
 MangleContext *ASTContext::createMangleContext(const TargetInfo *T) {
   if (!T)
     T = Target;
   switch (T->getCXXABI().getKind()) {
   case TargetCXXABI::AppleARM64:
   case TargetCXXABI::Fuchsia:
   case TargetCXXABI::GenericAArch64:
   case TargetCXXABI::GenericItanium:
   case TargetCXXABI::GenericARM:
   case TargetCXXABI::GenericMIPS:
   case TargetCXXABI::iOS:
   case TargetCXXABI::WebAssembly:
   case TargetCXXABI::WatchOS:
   case TargetCXXABI::XL:
     return ItaniumMangleContext::create(*this, getDiagnostics());
   case TargetCXXABI::Microsoft:
     return MicrosoftMangleContext::create(*this, getDiagnostics());
   }
   llvm_unreachable("Unsupported ABI");
 }
 
 MangleContext *ASTContext::createDeviceMangleContext(const TargetInfo &T) {
   assert(T.getCXXABI().getKind() != TargetCXXABI::Microsoft &&
          "Device mangle context does not support Microsoft mangling.");
   switch (T.getCXXABI().getKind()) {
   case TargetCXXABI::AppleARM64:
   case TargetCXXABI::Fuchsia:
   case TargetCXXABI::GenericAArch64:
   case TargetCXXABI::GenericItanium:
   case TargetCXXABI::GenericARM:
   case TargetCXXABI::GenericMIPS:
   case TargetCXXABI::iOS:
   case TargetCXXABI::WebAssembly:
   case TargetCXXABI::WatchOS:
   case TargetCXXABI::XL:
     return ItaniumMangleContext::create(
         *this, getDiagnostics(),
         [](ASTContext &, const NamedDecl *ND) -> llvm::Optional<unsigned> {
           if (const auto *RD = dyn_cast<CXXRecordDecl>(ND))
             return RD->getDeviceLambdaManglingNumber();
           return llvm::None;
         });
   case TargetCXXABI::Microsoft:
     return MicrosoftMangleContext::create(*this, getDiagnostics());
   }
   llvm_unreachable("Unsupported ABI");
 }
 
 CXXABI::~CXXABI() = default;
 
 size_t ASTContext::getSideTableAllocatedMemory() const {
   return ASTRecordLayouts.getMemorySize() +
          llvm::capacity_in_bytes(ObjCLayouts) +
          llvm::capacity_in_bytes(KeyFunctions) +
          llvm::capacity_in_bytes(ObjCImpls) +
          llvm::capacity_in_bytes(BlockVarCopyInits) +
          llvm::capacity_in_bytes(DeclAttrs) +
          llvm::capacity_in_bytes(TemplateOrInstantiation) +
          llvm::capacity_in_bytes(InstantiatedFromUsingDecl) +
          llvm::capacity_in_bytes(InstantiatedFromUsingShadowDecl) +
          llvm::capacity_in_bytes(InstantiatedFromUnnamedFieldDecl) +
          llvm::capacity_in_bytes(OverriddenMethods) +
          llvm::capacity_in_bytes(Types) +
          llvm::capacity_in_bytes(VariableArrayTypes);
 }
 
 /// getIntTypeForBitwidth -
 /// sets integer QualTy according to specified details:
 /// bitwidth, signed/unsigned.
 /// Returns empty type if there is no appropriate target types.
 QualType ASTContext::getIntTypeForBitwidth(unsigned DestWidth,
                                            unsigned Signed) const {
   TargetInfo::IntType Ty = getTargetInfo().getIntTypeByWidth(DestWidth, Signed);
   CanQualType QualTy = getFromTargetType(Ty);
   if (!QualTy && DestWidth == 128)
     return Signed ? Int128Ty : UnsignedInt128Ty;
   return QualTy;
 }
 
 /// getRealTypeForBitwidth -
 /// sets floating point QualTy according to specified bitwidth.
 /// Returns empty type if there is no appropriate target types.
 QualType ASTContext::getRealTypeForBitwidth(unsigned DestWidth,
                                             FloatModeKind ExplicitType) const {
   FloatModeKind Ty =
       getTargetInfo().getRealTypeByWidth(DestWidth, ExplicitType);
   switch (Ty) {
   case FloatModeKind::Float:
     return FloatTy;
   case FloatModeKind::Double:
     return DoubleTy;
   case FloatModeKind::LongDouble:
     return LongDoubleTy;
   case FloatModeKind::Float128:
     return Float128Ty;
   case FloatModeKind::Ibm128:
     return Ibm128Ty;
   case FloatModeKind::NoFloat:
     return {};
   }
 
   llvm_unreachable("Unhandled TargetInfo::RealType value");
 }
 
 void ASTContext::setManglingNumber(const NamedDecl *ND, unsigned Number) {
   if (Number > 1)
     MangleNumbers[ND] = Number;
 }
 
 unsigned ASTContext::getManglingNumber(const NamedDecl *ND) const {
   auto I = MangleNumbers.find(ND);
   return I != MangleNumbers.end() ? I->second : 1;
 }
 
 void ASTContext::setStaticLocalNumber(const VarDecl *VD, unsigned Number) {
   if (Number > 1)
     StaticLocalNumbers[VD] = Number;
 }
 
 unsigned ASTContext::getStaticLocalNumber(const VarDecl *VD) const {
   auto I = StaticLocalNumbers.find(VD);
   return I != StaticLocalNumbers.end() ? I->second : 1;
 }
 
 MangleNumberingContext &
 ASTContext::getManglingNumberContext(const DeclContext *DC) {
   assert(LangOpts.CPlusPlus);  // We don't need mangling numbers for plain C.
   std::unique_ptr<MangleNumberingContext> &MCtx = MangleNumberingContexts[DC];
   if (!MCtx)
     MCtx = createMangleNumberingContext();
   return *MCtx;
 }
 
 MangleNumberingContext &
 ASTContext::getManglingNumberContext(NeedExtraManglingDecl_t, const Decl *D) {
   assert(LangOpts.CPlusPlus); // We don't need mangling numbers for plain C.
   std::unique_ptr<MangleNumberingContext> &MCtx =
       ExtraMangleNumberingContexts[D];
   if (!MCtx)
     MCtx = createMangleNumberingContext();
   return *MCtx;
 }
 
 std::unique_ptr<MangleNumberingContext>
 ASTContext::createMangleNumberingContext() const {
   return ABI->createMangleNumberingContext();
 }
 
 const CXXConstructorDecl *
 ASTContext::getCopyConstructorForExceptionObject(CXXRecordDecl *RD) {
   return ABI->getCopyConstructorForExceptionObject(
       cast<CXXRecordDecl>(RD->getFirstDecl()));
 }
 
 void ASTContext::addCopyConstructorForExceptionObject(CXXRecordDecl *RD,
                                                       CXXConstructorDecl *CD) {
   return ABI->addCopyConstructorForExceptionObject(
       cast<CXXRecordDecl>(RD->getFirstDecl()),
       cast<CXXConstructorDecl>(CD->getFirstDecl()));
 }
 
 void ASTContext::addTypedefNameForUnnamedTagDecl(TagDecl *TD,
                                                  TypedefNameDecl *DD) {
   return ABI->addTypedefNameForUnnamedTagDecl(TD, DD);
 }
 
 TypedefNameDecl *
 ASTContext::getTypedefNameForUnnamedTagDecl(const TagDecl *TD) {
   return ABI->getTypedefNameForUnnamedTagDecl(TD);
 }
 
 void ASTContext::addDeclaratorForUnnamedTagDecl(TagDecl *TD,
                                                 DeclaratorDecl *DD) {
   return ABI->addDeclaratorForUnnamedTagDecl(TD, DD);
 }
 
 DeclaratorDecl *ASTContext::getDeclaratorForUnnamedTagDecl(const TagDecl *TD) {
   return ABI->getDeclaratorForUnnamedTagDecl(TD);
 }
 
 void ASTContext::setParameterIndex(const ParmVarDecl *D, unsigned int index) {
   ParamIndices[D] = index;
 }
 
 unsigned ASTContext::getParameterIndex(const ParmVarDecl *D) const {
   ParameterIndexTable::const_iterator I = ParamIndices.find(D);
   assert(I != ParamIndices.end() &&
          "ParmIndices lacks entry set by ParmVarDecl");
   return I->second;
 }
 
 QualType ASTContext::getStringLiteralArrayType(QualType EltTy,
                                                unsigned Length) const {
   // A C++ string literal has a const-qualified element type (C++ 2.13.4p1).
   if (getLangOpts().CPlusPlus || getLangOpts().ConstStrings)
     EltTy = EltTy.withConst();
 
   EltTy = adjustStringLiteralBaseType(EltTy);
 
   // Get an array type for the string, according to C99 6.4.5. This includes
   // the null terminator character.
   return getConstantArrayType(EltTy, llvm::APInt(32, Length + 1), nullptr,
                               ArrayType::Normal, /*IndexTypeQuals*/ 0);
 }
 
 StringLiteral *
 ASTContext::getPredefinedStringLiteralFromCache(StringRef Key) const {
   StringLiteral *&Result = StringLiteralCache[Key];
   if (!Result)
     Result = StringLiteral::Create(
         *this, Key, StringLiteral::Ascii,
         /*Pascal*/ false, getStringLiteralArrayType(CharTy, Key.size()),
         SourceLocation());
   return Result;
 }
 
 MSGuidDecl *
 ASTContext::getMSGuidDecl(MSGuidDecl::Parts Parts) const {
   assert(MSGuidTagDecl && "building MS GUID without MS extensions?");
 
   llvm::FoldingSetNodeID ID;
   MSGuidDecl::Profile(ID, Parts);
 
   void *InsertPos;
   if (MSGuidDecl *Existing = MSGuidDecls.FindNodeOrInsertPos(ID, InsertPos))
     return Existing;
 
   QualType GUIDType = getMSGuidType().withConst();
   MSGuidDecl *New = MSGuidDecl::Create(*this, GUIDType, Parts);
   MSGuidDecls.InsertNode(New, InsertPos);
   return New;
 }
 
 TemplateParamObjectDecl *
 ASTContext::getTemplateParamObjectDecl(QualType T, const APValue &V) const {
   assert(T->isRecordType() && "template param object of unexpected type");
 
   // C++ [temp.param]p8:
   //   [...] a static storage duration object of type 'const T' [...]
   T.addConst();
 
   llvm::FoldingSetNodeID ID;
   TemplateParamObjectDecl::Profile(ID, T, V);
 
   void *InsertPos;
   if (TemplateParamObjectDecl *Existing =
           TemplateParamObjectDecls.FindNodeOrInsertPos(ID, InsertPos))
     return Existing;
 
   TemplateParamObjectDecl *New = TemplateParamObjectDecl::Create(*this, T, V);
   TemplateParamObjectDecls.InsertNode(New, InsertPos);
   return New;
 }
 
 bool ASTContext::AtomicUsesUnsupportedLibcall(const AtomicExpr *E) const {
   const llvm::Triple &T = getTargetInfo().getTriple();
   if (!T.isOSDarwin())
     return false;
 
   if (!(T.isiOS() && T.isOSVersionLT(7)) &&
       !(T.isMacOSX() && T.isOSVersionLT(10, 9)))
     return false;
 
   QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
   CharUnits sizeChars = getTypeSizeInChars(AtomicTy);
   uint64_t Size = sizeChars.getQuantity();
   CharUnits alignChars = getTypeAlignInChars(AtomicTy);
   unsigned Align = alignChars.getQuantity();
   unsigned MaxInlineWidthInBits = getTargetInfo().getMaxAtomicInlineWidth();
   return (Size != Align || toBits(sizeChars) > MaxInlineWidthInBits);
 }
 
 bool
 ASTContext::ObjCMethodsAreEqual(const ObjCMethodDecl *MethodDecl,
                                 const ObjCMethodDecl *MethodImpl) {
   // No point trying to match an unavailable/deprecated mothod.
   if (MethodDecl->hasAttr<UnavailableAttr>()
       || MethodDecl->hasAttr<DeprecatedAttr>())
     return false;
   if (MethodDecl->getObjCDeclQualifier() !=
       MethodImpl->getObjCDeclQualifier())
     return false;
   if (!hasSameType(MethodDecl->getReturnType(), MethodImpl->getReturnType()))
     return false;
 
   if (MethodDecl->param_size() != MethodImpl->param_size())
     return false;
 
   for (ObjCMethodDecl::param_const_iterator IM = MethodImpl->param_begin(),
        IF = MethodDecl->param_begin(), EM = MethodImpl->param_end(),
        EF = MethodDecl->param_end();
        IM != EM && IF != EF; ++IM, ++IF) {
     const ParmVarDecl *DeclVar = (*IF);
     const ParmVarDecl *ImplVar = (*IM);
     if (ImplVar->getObjCDeclQualifier() != DeclVar->getObjCDeclQualifier())
       return false;
     if (!hasSameType(DeclVar->getType(), ImplVar->getType()))
       return false;
   }
 
   return (MethodDecl->isVariadic() == MethodImpl->isVariadic());
 }
 
 uint64_t ASTContext::getTargetNullPointerValue(QualType QT) const {
   LangAS AS;
   if (QT->getUnqualifiedDesugaredType()->isNullPtrType())
     AS = LangAS::Default;
   else
     AS = QT->getPointeeType().getAddressSpace();
 
   return getTargetInfo().getNullPointerValue(AS);
 }
 
 unsigned ASTContext::getTargetAddressSpace(QualType T) const {
   return T->isFunctionType() ? getTargetInfo().getProgramAddressSpace()
                              : getTargetAddressSpace(T.getQualifiers());
 }
 
 unsigned ASTContext::getTargetAddressSpace(Qualifiers Q) const {
   return getTargetAddressSpace(Q.getAddressSpace());
 }
 
 unsigned ASTContext::getTargetAddressSpace(LangAS AS) const {
   if (isTargetAddressSpace(AS))
     return toTargetAddressSpace(AS);
   else
     return (*AddrSpaceMap)[(unsigned)AS];
 }
 
 QualType ASTContext::getCorrespondingSaturatedType(QualType Ty) const {
   assert(Ty->isFixedPointType());
 
   if (Ty->isSaturatedFixedPointType()) return Ty;
 
   switch (Ty->castAs<BuiltinType>()->getKind()) {
     default:
       llvm_unreachable("Not a fixed point type!");
     case BuiltinType::ShortAccum:
       return SatShortAccumTy;
     case BuiltinType::Accum:
       return SatAccumTy;
     case BuiltinType::LongAccum:
       return SatLongAccumTy;
     case BuiltinType::UShortAccum:
       return SatUnsignedShortAccumTy;
     case BuiltinType::UAccum:
       return SatUnsignedAccumTy;
     case BuiltinType::ULongAccum:
       return SatUnsignedLongAccumTy;
     case BuiltinType::ShortFract:
       return SatShortFractTy;
     case BuiltinType::Fract:
       return SatFractTy;
     case BuiltinType::LongFract:
       return SatLongFractTy;
     case BuiltinType::UShortFract:
       return SatUnsignedShortFractTy;
     case BuiltinType::UFract:
       return SatUnsignedFractTy;
     case BuiltinType::ULongFract:
       return SatUnsignedLongFractTy;
   }
 }
 
 LangAS ASTContext::getLangASForBuiltinAddressSpace(unsigned AS) const {
   if (LangOpts.OpenCL)
     return getTargetInfo().getOpenCLBuiltinAddressSpace(AS);
 
   if (LangOpts.CUDA)
     return getTargetInfo().getCUDABuiltinAddressSpace(AS);
 
   return getLangASFromTargetAS(AS);
 }
 
 // Explicitly instantiate this in case a Redeclarable<T> is used from a TU that
 // doesn't include ASTContext.h
 template
 clang::LazyGenerationalUpdatePtr<
     const Decl *, Decl *, &ExternalASTSource::CompleteRedeclChain>::ValueType
 clang::LazyGenerationalUpdatePtr<
     const Decl *, Decl *, &ExternalASTSource::CompleteRedeclChain>::makeValue(
         const clang::ASTContext &Ctx, Decl *Value);
 
 unsigned char ASTContext::getFixedPointScale(QualType Ty) const {
   assert(Ty->isFixedPointType());
 
   const TargetInfo &Target = getTargetInfo();
   switch (Ty->castAs<BuiltinType>()->getKind()) {
     default:
       llvm_unreachable("Not a fixed point type!");
     case BuiltinType::ShortAccum:
     case BuiltinType::SatShortAccum:
       return Target.getShortAccumScale();
     case BuiltinType::Accum:
     case BuiltinType::SatAccum:
       return Target.getAccumScale();
     case BuiltinType::LongAccum:
     case BuiltinType::SatLongAccum:
       return Target.getLongAccumScale();
     case BuiltinType::UShortAccum:
     case BuiltinType::SatUShortAccum:
       return Target.getUnsignedShortAccumScale();
     case BuiltinType::UAccum:
     case BuiltinType::SatUAccum:
       return Target.getUnsignedAccumScale();
     case BuiltinType::ULongAccum:
     case BuiltinType::SatULongAccum:
       return Target.getUnsignedLongAccumScale();
     case BuiltinType::ShortFract:
     case BuiltinType::SatShortFract:
       return Target.getShortFractScale();
     case BuiltinType::Fract:
     case BuiltinType::SatFract:
       return Target.getFractScale();
     case BuiltinType::LongFract:
     case BuiltinType::SatLongFract:
       return Target.getLongFractScale();
     case BuiltinType::UShortFract:
     case BuiltinType::SatUShortFract:
       return Target.getUnsignedShortFractScale();
     case BuiltinType::UFract:
     case BuiltinType::SatUFract:
       return Target.getUnsignedFractScale();
     case BuiltinType::ULongFract:
     case BuiltinType::SatULongFract:
       return Target.getUnsignedLongFractScale();
   }
 }
 
 unsigned char ASTContext::getFixedPointIBits(QualType Ty) const {
   assert(Ty->isFixedPointType());
 
   const TargetInfo &Target = getTargetInfo();
   switch (Ty->castAs<BuiltinType>()->getKind()) {
     default:
       llvm_unreachable("Not a fixed point type!");
     case BuiltinType::ShortAccum:
     case BuiltinType::SatShortAccum:
       return Target.getShortAccumIBits();
     case BuiltinType::Accum:
     case BuiltinType::SatAccum:
       return Target.getAccumIBits();
     case BuiltinType::LongAccum:
     case BuiltinType::SatLongAccum:
       return Target.getLongAccumIBits();
     case BuiltinType::UShortAccum:
     case BuiltinType::SatUShortAccum:
       return Target.getUnsignedShortAccumIBits();
     case BuiltinType::UAccum:
     case BuiltinType::SatUAccum:
       return Target.getUnsignedAccumIBits();
     case BuiltinType::ULongAccum:
     case BuiltinType::SatULongAccum:
       return Target.getUnsignedLongAccumIBits();
     case BuiltinType::ShortFract:
     case BuiltinType::SatShortFract:
     case BuiltinType::Fract:
     case BuiltinType::SatFract:
     case BuiltinType::LongFract:
     case BuiltinType::SatLongFract:
     case BuiltinType::UShortFract:
     case BuiltinType::SatUShortFract:
     case BuiltinType::UFract:
     case BuiltinType::SatUFract:
     case BuiltinType::ULongFract:
     case BuiltinType::SatULongFract:
       return 0;
   }
 }
 
 llvm::FixedPointSemantics
 ASTContext::getFixedPointSemantics(QualType Ty) const {
   assert((Ty->isFixedPointType() || Ty->isIntegerType()) &&
          "Can only get the fixed point semantics for a "
          "fixed point or integer type.");
   if (Ty->isIntegerType())
     return llvm::FixedPointSemantics::GetIntegerSemantics(
         getIntWidth(Ty), Ty->isSignedIntegerType());
 
   bool isSigned = Ty->isSignedFixedPointType();
   return llvm::FixedPointSemantics(
       static_cast<unsigned>(getTypeSize(Ty)), getFixedPointScale(Ty), isSigned,
       Ty->isSaturatedFixedPointType(),
       !isSigned && getTargetInfo().doUnsignedFixedPointTypesHavePadding());
 }
 
 llvm::APFixedPoint ASTContext::getFixedPointMax(QualType Ty) const {
   assert(Ty->isFixedPointType());
   return llvm::APFixedPoint::getMax(getFixedPointSemantics(Ty));
 }
 
 llvm::APFixedPoint ASTContext::getFixedPointMin(QualType Ty) const {
   assert(Ty->isFixedPointType());
   return llvm::APFixedPoint::getMin(getFixedPointSemantics(Ty));
 }
 
 QualType ASTContext::getCorrespondingSignedFixedPointType(QualType Ty) const {
   assert(Ty->isUnsignedFixedPointType() &&
          "Expected unsigned fixed point type");
 
   switch (Ty->castAs<BuiltinType>()->getKind()) {
   case BuiltinType::UShortAccum:
     return ShortAccumTy;
   case BuiltinType::UAccum:
     return AccumTy;
   case BuiltinType::ULongAccum:
     return LongAccumTy;
   case BuiltinType::SatUShortAccum:
     return SatShortAccumTy;
   case BuiltinType::SatUAccum:
     return SatAccumTy;
   case BuiltinType::SatULongAccum:
     return SatLongAccumTy;
   case BuiltinType::UShortFract:
     return ShortFractTy;
   case BuiltinType::UFract:
     return FractTy;
   case BuiltinType::ULongFract:
     return LongFractTy;
   case BuiltinType::SatUShortFract:
     return SatShortFractTy;
   case BuiltinType::SatUFract:
     return SatFractTy;
   case BuiltinType::SatULongFract:
     return SatLongFractTy;
   default:
     llvm_unreachable("Unexpected unsigned fixed point type");
   }
 }
 
 ParsedTargetAttr
 ASTContext::filterFunctionTargetAttrs(const TargetAttr *TD) const {
   assert(TD != nullptr);
   ParsedTargetAttr ParsedAttr = TD->parse();
 
   llvm::erase_if(ParsedAttr.Features, [&](const std::string &Feat) {
     return !Target->isValidFeatureName(StringRef{Feat}.substr(1));
   });
   return ParsedAttr;
 }
 
 void ASTContext::getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap,
                                        const FunctionDecl *FD) const {
   if (FD)
     getFunctionFeatureMap(FeatureMap, GlobalDecl().getWithDecl(FD));
   else
     Target->initFeatureMap(FeatureMap, getDiagnostics(),
                            Target->getTargetOpts().CPU,
                            Target->getTargetOpts().Features);
 }
 
 // Fills in the supplied string map with the set of target features for the
 // passed in function.
 void ASTContext::getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap,
                                        GlobalDecl GD) const {
   StringRef TargetCPU = Target->getTargetOpts().CPU;
   const FunctionDecl *FD = GD.getDecl()->getAsFunction();
   if (const auto *TD = FD->getAttr<TargetAttr>()) {
     ParsedTargetAttr ParsedAttr = filterFunctionTargetAttrs(TD);
 
     // Make a copy of the features as passed on the command line into the
     // beginning of the additional features from the function to override.
     ParsedAttr.Features.insert(
         ParsedAttr.Features.begin(),
         Target->getTargetOpts().FeaturesAsWritten.begin(),
         Target->getTargetOpts().FeaturesAsWritten.end());
 
     if (ParsedAttr.Architecture != "" &&
         Target->isValidCPUName(ParsedAttr.Architecture))
       TargetCPU = ParsedAttr.Architecture;
 
     // Now populate the feature map, first with the TargetCPU which is either
     // the default or a new one from the target attribute string. Then we'll use
     // the passed in features (FeaturesAsWritten) along with the new ones from
     // the attribute.
     Target->initFeatureMap(FeatureMap, getDiagnostics(), TargetCPU,
                            ParsedAttr.Features);
   } else if (const auto *SD = FD->getAttr<CPUSpecificAttr>()) {
     llvm::SmallVector<StringRef, 32> FeaturesTmp;
     Target->getCPUSpecificCPUDispatchFeatures(
         SD->getCPUName(GD.getMultiVersionIndex())->getName(), FeaturesTmp);
     std::vector<std::string> Features(FeaturesTmp.begin(), FeaturesTmp.end());
     Features.insert(Features.begin(),
                     Target->getTargetOpts().FeaturesAsWritten.begin(),
                     Target->getTargetOpts().FeaturesAsWritten.end());
     Target->initFeatureMap(FeatureMap, getDiagnostics(), TargetCPU, Features);
   } else if (const auto *TC = FD->getAttr<TargetClonesAttr>()) {
     std::vector<std::string> Features;
     StringRef VersionStr = TC->getFeatureStr(GD.getMultiVersionIndex());
     if (VersionStr.startswith("arch="))
       TargetCPU = VersionStr.drop_front(sizeof("arch=") - 1);
     else if (VersionStr != "default")
       Features.push_back((StringRef{"+"} + VersionStr).str());
 
     Target->initFeatureMap(FeatureMap, getDiagnostics(), TargetCPU, Features);
   } else {
     FeatureMap = Target->getTargetOpts().FeatureMap;
   }
 }
 
 OMPTraitInfo &ASTContext::getNewOMPTraitInfo() {
   OMPTraitInfoVector.emplace_back(new OMPTraitInfo());
   return *OMPTraitInfoVector.back();
 }
 
 const StreamingDiagnostic &clang::
 operator<<(const StreamingDiagnostic &DB,
            const ASTContext::SectionInfo &Section) {
   if (Section.Decl)
     return DB << Section.Decl;
   return DB << "a prior #pragma section";
 }
 
 bool ASTContext::mayExternalizeStaticVar(const Decl *D) const {
   bool IsStaticVar =
       isa<VarDecl>(D) && cast<VarDecl>(D)->getStorageClass() == SC_Static;
   bool IsExplicitDeviceVar = (D->hasAttr<CUDADeviceAttr>() &&
                               !D->getAttr<CUDADeviceAttr>()->isImplicit()) ||
                              (D->hasAttr<CUDAConstantAttr>() &&
                               !D->getAttr<CUDAConstantAttr>()->isImplicit());
   // CUDA/HIP: static managed variables need to be externalized since it is
-  // a declaration in IR, therefore cannot have internal linkage.
-  return IsStaticVar &&
-         (D->hasAttr<HIPManagedAttr>() || IsExplicitDeviceVar);
+  // a declaration in IR, therefore cannot have internal linkage. Kernels in
+  // anonymous name space needs to be externalized to avoid duplicate symbols.
+  return (IsStaticVar &&
+          (D->hasAttr<HIPManagedAttr>() || IsExplicitDeviceVar)) ||
+         (D->hasAttr<CUDAGlobalAttr>() &&
+          basicGVALinkageForFunction(*this, cast<FunctionDecl>(D)) ==
+              GVA_Internal);
 }
 
 bool ASTContext::shouldExternalizeStaticVar(const Decl *D) const {
   return mayExternalizeStaticVar(D) &&
-         (D->hasAttr<HIPManagedAttr>() ||
+         (D->hasAttr<HIPManagedAttr>() || D->hasAttr<CUDAGlobalAttr>() ||
           CUDADeviceVarODRUsedByHost.count(cast<VarDecl>(D)));
 }
 
 StringRef ASTContext::getCUIDHash() const {
   if (!CUIDHash.empty())
     return CUIDHash;
   if (LangOpts.CUID.empty())
     return StringRef();
   CUIDHash = llvm::utohexstr(llvm::MD5Hash(LangOpts.CUID), /*LowerCase=*/true);
   return CUIDHash;
 }
diff --git a/contrib/llvm-project/clang/lib/AST/ItaniumMangle.cpp b/contrib/llvm-project/clang/lib/AST/ItaniumMangle.cpp
index 2e734e2b28cd..68d4d1271cdb 100644
--- a/contrib/llvm-project/clang/lib/AST/ItaniumMangle.cpp
+++ b/contrib/llvm-project/clang/lib/AST/ItaniumMangle.cpp
@@ -1,6437 +1,6463 @@
 //===--- ItaniumMangle.cpp - Itanium C++ Name Mangling ----------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // Implements C++ name mangling according to the Itanium C++ ABI,
 // which is used in GCC 3.2 and newer (and many compilers that are
 // ABI-compatible with GCC):
 //
 //   http://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclOpenMP.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprConcepts.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/Mangle.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/Basic/ABI.h"
 #include "clang/Basic/Module.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/Thunk.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
 
 namespace {
 
-/// Retrieve the declaration context that should be used when mangling the given
-/// declaration.
-static const DeclContext *getEffectiveDeclContext(const Decl *D) {
-  // The ABI assumes that lambda closure types that occur within
-  // default arguments live in the context of the function. However, due to
-  // the way in which Clang parses and creates function declarations, this is
-  // not the case: the lambda closure type ends up living in the context
-  // where the function itself resides, because the function declaration itself
-  // had not yet been created. Fix the context here.
-  if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(D)) {
-    if (RD->isLambda())
-      if (ParmVarDecl *ContextParam
-            = dyn_cast_or_null<ParmVarDecl>(RD->getLambdaContextDecl()))
-        return ContextParam->getDeclContext();
-  }
-
-  // Perform the same check for block literals.
-  if (const BlockDecl *BD = dyn_cast<BlockDecl>(D)) {
-    if (ParmVarDecl *ContextParam
-          = dyn_cast_or_null<ParmVarDecl>(BD->getBlockManglingContextDecl()))
-      return ContextParam->getDeclContext();
-  }
-
-  const DeclContext *DC = D->getDeclContext();
-  if (isa<CapturedDecl>(DC) || isa<OMPDeclareReductionDecl>(DC) ||
-      isa<OMPDeclareMapperDecl>(DC)) {
-    return getEffectiveDeclContext(cast<Decl>(DC));
-  }
-
-  if (const auto *VD = dyn_cast<VarDecl>(D))
-    if (VD->isExternC())
-      return VD->getASTContext().getTranslationUnitDecl();
-
-  if (const auto *FD = dyn_cast<FunctionDecl>(D))
-    if (FD->isExternC())
-      return FD->getASTContext().getTranslationUnitDecl();
-
-  return DC->getRedeclContext();
-}
-
-static const DeclContext *getEffectiveParentContext(const DeclContext *DC) {
-  return getEffectiveDeclContext(cast<Decl>(DC));
-}
-
 static bool isLocalContainerContext(const DeclContext *DC) {
   return isa<FunctionDecl>(DC) || isa<ObjCMethodDecl>(DC) || isa<BlockDecl>(DC);
 }
 
-static const RecordDecl *GetLocalClassDecl(const Decl *D) {
-  const DeclContext *DC = getEffectiveDeclContext(D);
-  while (!DC->isNamespace() && !DC->isTranslationUnit()) {
-    if (isLocalContainerContext(DC))
-      return dyn_cast<RecordDecl>(D);
-    D = cast<Decl>(DC);
-    DC = getEffectiveDeclContext(D);
-  }
-  return nullptr;
-}
-
 static const FunctionDecl *getStructor(const FunctionDecl *fn) {
   if (const FunctionTemplateDecl *ftd = fn->getPrimaryTemplate())
     return ftd->getTemplatedDecl();
 
   return fn;
 }
 
 static const NamedDecl *getStructor(const NamedDecl *decl) {
   const FunctionDecl *fn = dyn_cast_or_null<FunctionDecl>(decl);
   return (fn ? getStructor(fn) : decl);
 }
 
 static bool isLambda(const NamedDecl *ND) {
   const CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>(ND);
   if (!Record)
     return false;
 
   return Record->isLambda();
 }
 
 static const unsigned UnknownArity = ~0U;
 
 class ItaniumMangleContextImpl : public ItaniumMangleContext {
   typedef std::pair<const DeclContext*, IdentifierInfo*> DiscriminatorKeyTy;
   llvm::DenseMap<DiscriminatorKeyTy, unsigned> Discriminator;
   llvm::DenseMap<const NamedDecl*, unsigned> Uniquifier;
   const DiscriminatorOverrideTy DiscriminatorOverride = nullptr;
+  NamespaceDecl *StdNamespace = nullptr;
 
   bool NeedsUniqueInternalLinkageNames = false;
 
 public:
   explicit ItaniumMangleContextImpl(
       ASTContext &Context, DiagnosticsEngine &Diags,
       DiscriminatorOverrideTy DiscriminatorOverride)
       : ItaniumMangleContext(Context, Diags),
         DiscriminatorOverride(DiscriminatorOverride) {}
 
   /// @name Mangler Entry Points
   /// @{
 
   bool shouldMangleCXXName(const NamedDecl *D) override;
   bool shouldMangleStringLiteral(const StringLiteral *) override {
     return false;
   }
 
   bool isUniqueInternalLinkageDecl(const NamedDecl *ND) override;
   void needsUniqueInternalLinkageNames() override {
     NeedsUniqueInternalLinkageNames = true;
   }
 
   void mangleCXXName(GlobalDecl GD, raw_ostream &) override;
   void mangleThunk(const CXXMethodDecl *MD, const ThunkInfo &Thunk,
                    raw_ostream &) override;
   void mangleCXXDtorThunk(const CXXDestructorDecl *DD, CXXDtorType Type,
                           const ThisAdjustment &ThisAdjustment,
                           raw_ostream &) override;
   void mangleReferenceTemporary(const VarDecl *D, unsigned ManglingNumber,
                                 raw_ostream &) override;
   void mangleCXXVTable(const CXXRecordDecl *RD, raw_ostream &) override;
   void mangleCXXVTT(const CXXRecordDecl *RD, raw_ostream &) override;
   void mangleCXXCtorVTable(const CXXRecordDecl *RD, int64_t Offset,
                            const CXXRecordDecl *Type, raw_ostream &) override;
   void mangleCXXRTTI(QualType T, raw_ostream &) override;
   void mangleCXXRTTIName(QualType T, raw_ostream &) override;
   void mangleTypeName(QualType T, raw_ostream &) override;
 
   void mangleCXXCtorComdat(const CXXConstructorDecl *D, raw_ostream &) override;
   void mangleCXXDtorComdat(const CXXDestructorDecl *D, raw_ostream &) override;
   void mangleStaticGuardVariable(const VarDecl *D, raw_ostream &) override;
   void mangleDynamicInitializer(const VarDecl *D, raw_ostream &Out) override;
   void mangleDynamicAtExitDestructor(const VarDecl *D,
                                      raw_ostream &Out) override;
   void mangleDynamicStermFinalizer(const VarDecl *D, raw_ostream &Out) override;
   void mangleSEHFilterExpression(const NamedDecl *EnclosingDecl,
                                  raw_ostream &Out) override;
   void mangleSEHFinallyBlock(const NamedDecl *EnclosingDecl,
                              raw_ostream &Out) override;
   void mangleItaniumThreadLocalInit(const VarDecl *D, raw_ostream &) override;
   void mangleItaniumThreadLocalWrapper(const VarDecl *D,
                                        raw_ostream &) override;
 
   void mangleStringLiteral(const StringLiteral *, raw_ostream &) override;
 
   void mangleLambdaSig(const CXXRecordDecl *Lambda, raw_ostream &) override;
 
   bool getNextDiscriminator(const NamedDecl *ND, unsigned &disc) {
     // Lambda closure types are already numbered.
     if (isLambda(ND))
       return false;
 
     // Anonymous tags are already numbered.
     if (const TagDecl *Tag = dyn_cast<TagDecl>(ND)) {
       if (Tag->getName().empty() && !Tag->getTypedefNameForAnonDecl())
         return false;
     }
 
     // Use the canonical number for externally visible decls.
     if (ND->isExternallyVisible()) {
       unsigned discriminator = getASTContext().getManglingNumber(ND);
       if (discriminator == 1)
         return false;
       disc = discriminator - 2;
       return true;
     }
 
     // Make up a reasonable number for internal decls.
     unsigned &discriminator = Uniquifier[ND];
     if (!discriminator) {
       const DeclContext *DC = getEffectiveDeclContext(ND);
       discriminator = ++Discriminator[std::make_pair(DC, ND->getIdentifier())];
     }
     if (discriminator == 1)
       return false;
     disc = discriminator-2;
     return true;
   }
 
   std::string getLambdaString(const CXXRecordDecl *Lambda) override {
     // This function matches the one in MicrosoftMangle, which returns
     // the string that is used in lambda mangled names.
     assert(Lambda->isLambda() && "RD must be a lambda!");
     std::string Name("<lambda");
     Decl *LambdaContextDecl = Lambda->getLambdaContextDecl();
     unsigned LambdaManglingNumber = Lambda->getLambdaManglingNumber();
     unsigned LambdaId;
     const ParmVarDecl *Parm = dyn_cast_or_null<ParmVarDecl>(LambdaContextDecl);
     const FunctionDecl *Func =
         Parm ? dyn_cast<FunctionDecl>(Parm->getDeclContext()) : nullptr;
 
     if (Func) {
       unsigned DefaultArgNo =
           Func->getNumParams() - Parm->getFunctionScopeIndex();
       Name += llvm::utostr(DefaultArgNo);
       Name += "_";
     }
 
     if (LambdaManglingNumber)
       LambdaId = LambdaManglingNumber;
     else
       LambdaId = getAnonymousStructIdForDebugInfo(Lambda);
 
     Name += llvm::utostr(LambdaId);
     Name += '>';
     return Name;
   }
 
   DiscriminatorOverrideTy getDiscriminatorOverride() const override {
     return DiscriminatorOverride;
   }
 
+  NamespaceDecl *getStdNamespace();
+
+  const DeclContext *getEffectiveDeclContext(const Decl *D);
+  const DeclContext *getEffectiveParentContext(const DeclContext *DC) {
+    return getEffectiveDeclContext(cast<Decl>(DC));
+  }
+
+  bool isInternalLinkageDecl(const NamedDecl *ND);
+  const DeclContext *IgnoreLinkageSpecDecls(const DeclContext *DC);
+
   /// @}
 };
 
 /// Manage the mangling of a single name.
 class CXXNameMangler {
   ItaniumMangleContextImpl &Context;
   raw_ostream &Out;
   bool NullOut = false;
   /// In the "DisableDerivedAbiTags" mode derived ABI tags are not calculated.
   /// This mode is used when mangler creates another mangler recursively to
   /// calculate ABI tags for the function return value or the variable type.
   /// Also it is required to avoid infinite recursion in some cases.
   bool DisableDerivedAbiTags = false;
 
   /// The "structor" is the top-level declaration being mangled, if
   /// that's not a template specialization; otherwise it's the pattern
   /// for that specialization.
   const NamedDecl *Structor;
   unsigned StructorType;
 
   /// The next substitution sequence number.
   unsigned SeqID;
 
   class FunctionTypeDepthState {
     unsigned Bits;
 
     enum { InResultTypeMask = 1 };
 
   public:
     FunctionTypeDepthState() : Bits(0) {}
 
     /// The number of function types we're inside.
     unsigned getDepth() const {
       return Bits >> 1;
     }
 
     /// True if we're in the return type of the innermost function type.
     bool isInResultType() const {
       return Bits & InResultTypeMask;
     }
 
     FunctionTypeDepthState push() {
       FunctionTypeDepthState tmp = *this;
       Bits = (Bits & ~InResultTypeMask) + 2;
       return tmp;
     }
 
     void enterResultType() {
       Bits |= InResultTypeMask;
     }
 
     void leaveResultType() {
       Bits &= ~InResultTypeMask;
     }
 
     void pop(FunctionTypeDepthState saved) {
       assert(getDepth() == saved.getDepth() + 1);
       Bits = saved.Bits;
     }
 
   } FunctionTypeDepth;
 
   // abi_tag is a gcc attribute, taking one or more strings called "tags".
   // The goal is to annotate against which version of a library an object was
   // built and to be able to provide backwards compatibility ("dual abi").
   // For more information see docs/ItaniumMangleAbiTags.rst.
   typedef SmallVector<StringRef, 4> AbiTagList;
 
   // State to gather all implicit and explicit tags used in a mangled name.
   // Must always have an instance of this while emitting any name to keep
   // track.
   class AbiTagState final {
   public:
     explicit AbiTagState(AbiTagState *&Head) : LinkHead(Head) {
       Parent = LinkHead;
       LinkHead = this;
     }
 
     // No copy, no move.
     AbiTagState(const AbiTagState &) = delete;
     AbiTagState &operator=(const AbiTagState &) = delete;
 
     ~AbiTagState() { pop(); }
 
     void write(raw_ostream &Out, const NamedDecl *ND,
                const AbiTagList *AdditionalAbiTags) {
       ND = cast<NamedDecl>(ND->getCanonicalDecl());
       if (!isa<FunctionDecl>(ND) && !isa<VarDecl>(ND)) {
         assert(
             !AdditionalAbiTags &&
             "only function and variables need a list of additional abi tags");
         if (const auto *NS = dyn_cast<NamespaceDecl>(ND)) {
           if (const auto *AbiTag = NS->getAttr<AbiTagAttr>()) {
             UsedAbiTags.insert(UsedAbiTags.end(), AbiTag->tags().begin(),
                                AbiTag->tags().end());
           }
           // Don't emit abi tags for namespaces.
           return;
         }
       }
 
       AbiTagList TagList;
       if (const auto *AbiTag = ND->getAttr<AbiTagAttr>()) {
         UsedAbiTags.insert(UsedAbiTags.end(), AbiTag->tags().begin(),
                            AbiTag->tags().end());
         TagList.insert(TagList.end(), AbiTag->tags().begin(),
                        AbiTag->tags().end());
       }
 
       if (AdditionalAbiTags) {
         UsedAbiTags.insert(UsedAbiTags.end(), AdditionalAbiTags->begin(),
                            AdditionalAbiTags->end());
         TagList.insert(TagList.end(), AdditionalAbiTags->begin(),
                        AdditionalAbiTags->end());
       }
 
       llvm::sort(TagList);
       TagList.erase(std::unique(TagList.begin(), TagList.end()), TagList.end());
 
       writeSortedUniqueAbiTags(Out, TagList);
     }
 
     const AbiTagList &getUsedAbiTags() const { return UsedAbiTags; }
     void setUsedAbiTags(const AbiTagList &AbiTags) {
       UsedAbiTags = AbiTags;
     }
 
     const AbiTagList &getEmittedAbiTags() const {
       return EmittedAbiTags;
     }
 
     const AbiTagList &getSortedUniqueUsedAbiTags() {
       llvm::sort(UsedAbiTags);
       UsedAbiTags.erase(std::unique(UsedAbiTags.begin(), UsedAbiTags.end()),
                         UsedAbiTags.end());
       return UsedAbiTags;
     }
 
   private:
     //! All abi tags used implicitly or explicitly.
     AbiTagList UsedAbiTags;
     //! All explicit abi tags (i.e. not from namespace).
     AbiTagList EmittedAbiTags;
 
     AbiTagState *&LinkHead;
     AbiTagState *Parent = nullptr;
 
     void pop() {
       assert(LinkHead == this &&
              "abi tag link head must point to us on destruction");
       if (Parent) {
         Parent->UsedAbiTags.insert(Parent->UsedAbiTags.end(),
                                    UsedAbiTags.begin(), UsedAbiTags.end());
         Parent->EmittedAbiTags.insert(Parent->EmittedAbiTags.end(),
                                       EmittedAbiTags.begin(),
                                       EmittedAbiTags.end());
       }
       LinkHead = Parent;
     }
 
     void writeSortedUniqueAbiTags(raw_ostream &Out, const AbiTagList &AbiTags) {
       for (const auto &Tag : AbiTags) {
         EmittedAbiTags.push_back(Tag);
         Out << "B";
         Out << Tag.size();
         Out << Tag;
       }
     }
   };
 
   AbiTagState *AbiTags = nullptr;
   AbiTagState AbiTagsRoot;
 
   llvm::DenseMap<uintptr_t, unsigned> Substitutions;
   llvm::DenseMap<StringRef, unsigned> ModuleSubstitutions;
 
   ASTContext &getASTContext() const { return Context.getASTContext(); }
 
+  bool isStd(const NamespaceDecl *NS);
+  bool isStdNamespace(const DeclContext *DC);
+
+  const RecordDecl *GetLocalClassDecl(const Decl *D);
+  const DeclContext *IgnoreLinkageSpecDecls(const DeclContext *DC);
+  bool isSpecializedAs(QualType S, llvm::StringRef Name, QualType A);
+  bool isStdCharSpecialization(const ClassTemplateSpecializationDecl *SD,
+                               llvm::StringRef Name, bool HasAllocator);
+
 public:
   CXXNameMangler(ItaniumMangleContextImpl &C, raw_ostream &Out_,
                  const NamedDecl *D = nullptr, bool NullOut_ = false)
     : Context(C), Out(Out_), NullOut(NullOut_),  Structor(getStructor(D)),
       StructorType(0), SeqID(0), AbiTagsRoot(AbiTags) {
     // These can't be mangled without a ctor type or dtor type.
     assert(!D || (!isa<CXXDestructorDecl>(D) &&
                   !isa<CXXConstructorDecl>(D)));
   }
   CXXNameMangler(ItaniumMangleContextImpl &C, raw_ostream &Out_,
                  const CXXConstructorDecl *D, CXXCtorType Type)
     : Context(C), Out(Out_), Structor(getStructor(D)), StructorType(Type),
       SeqID(0), AbiTagsRoot(AbiTags) { }
   CXXNameMangler(ItaniumMangleContextImpl &C, raw_ostream &Out_,
                  const CXXDestructorDecl *D, CXXDtorType Type)
     : Context(C), Out(Out_), Structor(getStructor(D)), StructorType(Type),
       SeqID(0), AbiTagsRoot(AbiTags) { }
 
   CXXNameMangler(CXXNameMangler &Outer, raw_ostream &Out_)
       : Context(Outer.Context), Out(Out_), NullOut(false),
         Structor(Outer.Structor), StructorType(Outer.StructorType),
         SeqID(Outer.SeqID), FunctionTypeDepth(Outer.FunctionTypeDepth),
         AbiTagsRoot(AbiTags), Substitutions(Outer.Substitutions) {}
 
   CXXNameMangler(CXXNameMangler &Outer, llvm::raw_null_ostream &Out_)
       : Context(Outer.Context), Out(Out_), NullOut(true),
         Structor(Outer.Structor), StructorType(Outer.StructorType),
         SeqID(Outer.SeqID), FunctionTypeDepth(Outer.FunctionTypeDepth),
         AbiTagsRoot(AbiTags), Substitutions(Outer.Substitutions) {}
 
   raw_ostream &getStream() { return Out; }
 
   void disableDerivedAbiTags() { DisableDerivedAbiTags = true; }
   static bool shouldHaveAbiTags(ItaniumMangleContextImpl &C, const VarDecl *VD);
 
   void mangle(GlobalDecl GD);
   void mangleCallOffset(int64_t NonVirtual, int64_t Virtual);
   void mangleNumber(const llvm::APSInt &I);
   void mangleNumber(int64_t Number);
   void mangleFloat(const llvm::APFloat &F);
   void mangleFunctionEncoding(GlobalDecl GD);
   void mangleSeqID(unsigned SeqID);
   void mangleName(GlobalDecl GD);
   void mangleType(QualType T);
   void mangleNameOrStandardSubstitution(const NamedDecl *ND);
   void mangleLambdaSig(const CXXRecordDecl *Lambda);
 
 private:
 
   bool mangleSubstitution(const NamedDecl *ND);
   bool mangleSubstitution(QualType T);
   bool mangleSubstitution(TemplateName Template);
   bool mangleSubstitution(uintptr_t Ptr);
 
   void mangleExistingSubstitution(TemplateName name);
 
   bool mangleStandardSubstitution(const NamedDecl *ND);
 
   void addSubstitution(const NamedDecl *ND) {
     ND = cast<NamedDecl>(ND->getCanonicalDecl());
 
     addSubstitution(reinterpret_cast<uintptr_t>(ND));
   }
   void addSubstitution(QualType T);
   void addSubstitution(TemplateName Template);
   void addSubstitution(uintptr_t Ptr);
   // Destructive copy substitutions from other mangler.
   void extendSubstitutions(CXXNameMangler* Other);
 
   void mangleUnresolvedPrefix(NestedNameSpecifier *qualifier,
                               bool recursive = false);
   void mangleUnresolvedName(NestedNameSpecifier *qualifier,
                             DeclarationName name,
                             const TemplateArgumentLoc *TemplateArgs,
                             unsigned NumTemplateArgs,
                             unsigned KnownArity = UnknownArity);
 
   void mangleFunctionEncodingBareType(const FunctionDecl *FD);
 
   void mangleNameWithAbiTags(GlobalDecl GD,
                              const AbiTagList *AdditionalAbiTags);
   void mangleModuleName(const Module *M);
   void mangleModuleNamePrefix(StringRef Name);
   void mangleTemplateName(const TemplateDecl *TD,
                           const TemplateArgument *TemplateArgs,
                           unsigned NumTemplateArgs);
   void mangleUnqualifiedName(GlobalDecl GD,
                              const AbiTagList *AdditionalAbiTags) {
     mangleUnqualifiedName(GD, cast<NamedDecl>(GD.getDecl())->getDeclName(), UnknownArity,
                           AdditionalAbiTags);
   }
   void mangleUnqualifiedName(GlobalDecl GD, DeclarationName Name,
                              unsigned KnownArity,
                              const AbiTagList *AdditionalAbiTags);
   void mangleUnscopedName(GlobalDecl GD,
                           const AbiTagList *AdditionalAbiTags);
   void mangleUnscopedTemplateName(GlobalDecl GD,
                                   const AbiTagList *AdditionalAbiTags);
   void mangleSourceName(const IdentifierInfo *II);
   void mangleRegCallName(const IdentifierInfo *II);
   void mangleDeviceStubName(const IdentifierInfo *II);
   void mangleSourceNameWithAbiTags(
       const NamedDecl *ND, const AbiTagList *AdditionalAbiTags = nullptr);
   void mangleLocalName(GlobalDecl GD,
                        const AbiTagList *AdditionalAbiTags);
   void mangleBlockForPrefix(const BlockDecl *Block);
   void mangleUnqualifiedBlock(const BlockDecl *Block);
   void mangleTemplateParamDecl(const NamedDecl *Decl);
   void mangleLambda(const CXXRecordDecl *Lambda);
   void mangleNestedName(GlobalDecl GD, const DeclContext *DC,
                         const AbiTagList *AdditionalAbiTags,
                         bool NoFunction=false);
   void mangleNestedName(const TemplateDecl *TD,
                         const TemplateArgument *TemplateArgs,
                         unsigned NumTemplateArgs);
   void mangleNestedNameWithClosurePrefix(GlobalDecl GD,
                                          const NamedDecl *PrefixND,
                                          const AbiTagList *AdditionalAbiTags);
   void manglePrefix(NestedNameSpecifier *qualifier);
   void manglePrefix(const DeclContext *DC, bool NoFunction=false);
   void manglePrefix(QualType type);
   void mangleTemplatePrefix(GlobalDecl GD, bool NoFunction=false);
   void mangleTemplatePrefix(TemplateName Template);
   const NamedDecl *getClosurePrefix(const Decl *ND);
   void mangleClosurePrefix(const NamedDecl *ND, bool NoFunction = false);
   bool mangleUnresolvedTypeOrSimpleId(QualType DestroyedType,
                                       StringRef Prefix = "");
   void mangleOperatorName(DeclarationName Name, unsigned Arity);
   void mangleOperatorName(OverloadedOperatorKind OO, unsigned Arity);
   void mangleVendorQualifier(StringRef qualifier);
   void mangleQualifiers(Qualifiers Quals, const DependentAddressSpaceType *DAST = nullptr);
   void mangleRefQualifier(RefQualifierKind RefQualifier);
 
   void mangleObjCMethodName(const ObjCMethodDecl *MD);
 
   // Declare manglers for every type class.
 #define ABSTRACT_TYPE(CLASS, PARENT)
 #define NON_CANONICAL_TYPE(CLASS, PARENT)
 #define TYPE(CLASS, PARENT) void mangleType(const CLASS##Type *T);
 #include "clang/AST/TypeNodes.inc"
 
   void mangleType(const TagType*);
   void mangleType(TemplateName);
   static StringRef getCallingConvQualifierName(CallingConv CC);
   void mangleExtParameterInfo(FunctionProtoType::ExtParameterInfo info);
   void mangleExtFunctionInfo(const FunctionType *T);
   void mangleBareFunctionType(const FunctionProtoType *T, bool MangleReturnType,
                               const FunctionDecl *FD = nullptr);
   void mangleNeonVectorType(const VectorType *T);
   void mangleNeonVectorType(const DependentVectorType *T);
   void mangleAArch64NeonVectorType(const VectorType *T);
   void mangleAArch64NeonVectorType(const DependentVectorType *T);
   void mangleAArch64FixedSveVectorType(const VectorType *T);
   void mangleAArch64FixedSveVectorType(const DependentVectorType *T);
 
   void mangleIntegerLiteral(QualType T, const llvm::APSInt &Value);
   void mangleFloatLiteral(QualType T, const llvm::APFloat &V);
   void mangleFixedPointLiteral();
   void mangleNullPointer(QualType T);
 
   void mangleMemberExprBase(const Expr *base, bool isArrow);
   void mangleMemberExpr(const Expr *base, bool isArrow,
                         NestedNameSpecifier *qualifier,
                         NamedDecl *firstQualifierLookup,
                         DeclarationName name,
                         const TemplateArgumentLoc *TemplateArgs,
                         unsigned NumTemplateArgs,
                         unsigned knownArity);
   void mangleCastExpression(const Expr *E, StringRef CastEncoding);
   void mangleInitListElements(const InitListExpr *InitList);
   void mangleExpression(const Expr *E, unsigned Arity = UnknownArity,
                         bool AsTemplateArg = false);
   void mangleCXXCtorType(CXXCtorType T, const CXXRecordDecl *InheritedFrom);
   void mangleCXXDtorType(CXXDtorType T);
 
   void mangleTemplateArgs(TemplateName TN,
                           const TemplateArgumentLoc *TemplateArgs,
                           unsigned NumTemplateArgs);
   void mangleTemplateArgs(TemplateName TN, const TemplateArgument *TemplateArgs,
                           unsigned NumTemplateArgs);
   void mangleTemplateArgs(TemplateName TN, const TemplateArgumentList &AL);
   void mangleTemplateArg(TemplateArgument A, bool NeedExactType);
   void mangleTemplateArgExpr(const Expr *E);
   void mangleValueInTemplateArg(QualType T, const APValue &V, bool TopLevel,
                                 bool NeedExactType = false);
 
   void mangleTemplateParameter(unsigned Depth, unsigned Index);
 
   void mangleFunctionParam(const ParmVarDecl *parm);
 
   void writeAbiTags(const NamedDecl *ND,
                     const AbiTagList *AdditionalAbiTags);
 
   // Returns sorted unique list of ABI tags.
   AbiTagList makeFunctionReturnTypeTags(const FunctionDecl *FD);
   // Returns sorted unique list of ABI tags.
   AbiTagList makeVariableTypeTags(const VarDecl *VD);
 };
 
 }
 
-static bool isInternalLinkageDecl(const NamedDecl *ND) {
+NamespaceDecl *ItaniumMangleContextImpl::getStdNamespace() {
+  if (!StdNamespace) {
+    StdNamespace = NamespaceDecl::Create(
+        getASTContext(), getASTContext().getTranslationUnitDecl(),
+        /*Inline*/ false, SourceLocation(), SourceLocation(),
+        &getASTContext().Idents.get("std"),
+        /*PrevDecl*/ nullptr);
+    StdNamespace->setImplicit();
+  }
+  return StdNamespace;
+}
+
+/// Retrieve the declaration context that should be used when mangling the given
+/// declaration.
+const DeclContext *
+ItaniumMangleContextImpl::getEffectiveDeclContext(const Decl *D) {
+  // The ABI assumes that lambda closure types that occur within
+  // default arguments live in the context of the function. However, due to
+  // the way in which Clang parses and creates function declarations, this is
+  // not the case: the lambda closure type ends up living in the context
+  // where the function itself resides, because the function declaration itself
+  // had not yet been created. Fix the context here.
+  if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(D)) {
+    if (RD->isLambda())
+      if (ParmVarDecl *ContextParam =
+              dyn_cast_or_null<ParmVarDecl>(RD->getLambdaContextDecl()))
+        return ContextParam->getDeclContext();
+  }
+
+  // Perform the same check for block literals.
+  if (const BlockDecl *BD = dyn_cast<BlockDecl>(D)) {
+    if (ParmVarDecl *ContextParam =
+            dyn_cast_or_null<ParmVarDecl>(BD->getBlockManglingContextDecl()))
+      return ContextParam->getDeclContext();
+  }
+
+  // On ARM and AArch64, the va_list tag is always mangled as if in the std
+  // namespace. We do not represent va_list as actually being in the std
+  // namespace in C because this would result in incorrect debug info in C,
+  // among other things. It is important for both languages to have the same
+  // mangling in order for -fsanitize=cfi-icall to work.
+  if (D == getASTContext().getVaListTagDecl()) {
+    const llvm::Triple &T = getASTContext().getTargetInfo().getTriple();
+    if (T.isARM() || T.isThumb() || T.isAArch64())
+      return getStdNamespace();
+  }
+
+  const DeclContext *DC = D->getDeclContext();
+  if (isa<CapturedDecl>(DC) || isa<OMPDeclareReductionDecl>(DC) ||
+      isa<OMPDeclareMapperDecl>(DC)) {
+    return getEffectiveDeclContext(cast<Decl>(DC));
+  }
+
+  if (const auto *VD = dyn_cast<VarDecl>(D))
+    if (VD->isExternC())
+      return getASTContext().getTranslationUnitDecl();
+
+  if (const auto *FD = dyn_cast<FunctionDecl>(D))
+    if (FD->isExternC())
+      return getASTContext().getTranslationUnitDecl();
+
+  return DC->getRedeclContext();
+}
+
+bool ItaniumMangleContextImpl::isInternalLinkageDecl(const NamedDecl *ND) {
   if (ND && ND->getFormalLinkage() == InternalLinkage &&
       !ND->isExternallyVisible() &&
       getEffectiveDeclContext(ND)->isFileContext() &&
       !ND->isInAnonymousNamespace())
     return true;
   return false;
 }
 
 // Check if this Function Decl needs a unique internal linkage name.
 bool ItaniumMangleContextImpl::isUniqueInternalLinkageDecl(
     const NamedDecl *ND) {
   if (!NeedsUniqueInternalLinkageNames || !ND)
     return false;
 
   const auto *FD = dyn_cast<FunctionDecl>(ND);
   if (!FD)
     return false;
 
   // For C functions without prototypes, return false as their
   // names should not be mangled.
   if (!FD->getType()->getAs<FunctionProtoType>())
     return false;
 
   if (isInternalLinkageDecl(ND))
     return true;
 
   return false;
 }
 
 bool ItaniumMangleContextImpl::shouldMangleCXXName(const NamedDecl *D) {
   if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
     LanguageLinkage L = FD->getLanguageLinkage();
     // Overloadable functions need mangling.
     if (FD->hasAttr<OverloadableAttr>())
       return true;
 
     // "main" is not mangled.
     if (FD->isMain())
       return false;
 
     // The Windows ABI expects that we would never mangle "typical"
     // user-defined entry points regardless of visibility or freestanding-ness.
     //
     // N.B. This is distinct from asking about "main".  "main" has a lot of
     // special rules associated with it in the standard while these
     // user-defined entry points are outside of the purview of the standard.
     // For example, there can be only one definition for "main" in a standards
     // compliant program; however nothing forbids the existence of wmain and
     // WinMain in the same translation unit.
     if (FD->isMSVCRTEntryPoint())
       return false;
 
     // C++ functions and those whose names are not a simple identifier need
     // mangling.
     if (!FD->getDeclName().isIdentifier() || L == CXXLanguageLinkage)
       return true;
 
     // C functions are not mangled.
     if (L == CLanguageLinkage)
       return false;
   }
 
   // Otherwise, no mangling is done outside C++ mode.
   if (!getASTContext().getLangOpts().CPlusPlus)
     return false;
 
   if (const auto *VD = dyn_cast<VarDecl>(D)) {
     // Decompositions are mangled.
     if (isa<DecompositionDecl>(VD))
       return true;
 
     // C variables are not mangled.
     if (VD->isExternC())
       return false;
 
     // Variables at global scope with non-internal linkage are not mangled.
     const DeclContext *DC = getEffectiveDeclContext(D);
     // Check for extern variable declared locally.
     if (DC->isFunctionOrMethod() && D->hasLinkage())
       while (!DC->isFileContext())
         DC = getEffectiveParentContext(DC);
     if (DC->isTranslationUnit() && D->getFormalLinkage() != InternalLinkage &&
         !CXXNameMangler::shouldHaveAbiTags(*this, VD) &&
         !isa<VarTemplateSpecializationDecl>(VD))
       return false;
   }
 
   return true;
 }
 
 void CXXNameMangler::writeAbiTags(const NamedDecl *ND,
                                   const AbiTagList *AdditionalAbiTags) {
   assert(AbiTags && "require AbiTagState");
   AbiTags->write(Out, ND, DisableDerivedAbiTags ? nullptr : AdditionalAbiTags);
 }
 
 void CXXNameMangler::mangleSourceNameWithAbiTags(
     const NamedDecl *ND, const AbiTagList *AdditionalAbiTags) {
   mangleSourceName(ND->getIdentifier());
   writeAbiTags(ND, AdditionalAbiTags);
 }
 
 void CXXNameMangler::mangle(GlobalDecl GD) {
   // <mangled-name> ::= _Z <encoding>
   //            ::= <data name>
   //            ::= <special-name>
   Out << "_Z";
   if (isa<FunctionDecl>(GD.getDecl()))
     mangleFunctionEncoding(GD);
   else if (isa<VarDecl, FieldDecl, MSGuidDecl, TemplateParamObjectDecl,
                BindingDecl>(GD.getDecl()))
     mangleName(GD);
   else if (const IndirectFieldDecl *IFD =
                dyn_cast<IndirectFieldDecl>(GD.getDecl()))
     mangleName(IFD->getAnonField());
   else
     llvm_unreachable("unexpected kind of global decl");
 }
 
 void CXXNameMangler::mangleFunctionEncoding(GlobalDecl GD) {
   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
   // <encoding> ::= <function name> <bare-function-type>
 
   // Don't mangle in the type if this isn't a decl we should typically mangle.
   if (!Context.shouldMangleDeclName(FD)) {
     mangleName(GD);
     return;
   }
 
   AbiTagList ReturnTypeAbiTags = makeFunctionReturnTypeTags(FD);
   if (ReturnTypeAbiTags.empty()) {
     // There are no tags for return type, the simplest case.
     mangleName(GD);
     mangleFunctionEncodingBareType(FD);
     return;
   }
 
   // Mangle function name and encoding to temporary buffer.
   // We have to output name and encoding to the same mangler to get the same
   // substitution as it will be in final mangling.
   SmallString<256> FunctionEncodingBuf;
   llvm::raw_svector_ostream FunctionEncodingStream(FunctionEncodingBuf);
   CXXNameMangler FunctionEncodingMangler(*this, FunctionEncodingStream);
   // Output name of the function.
   FunctionEncodingMangler.disableDerivedAbiTags();
   FunctionEncodingMangler.mangleNameWithAbiTags(FD, nullptr);
 
   // Remember length of the function name in the buffer.
   size_t EncodingPositionStart = FunctionEncodingStream.str().size();
   FunctionEncodingMangler.mangleFunctionEncodingBareType(FD);
 
   // Get tags from return type that are not present in function name or
   // encoding.
   const AbiTagList &UsedAbiTags =
       FunctionEncodingMangler.AbiTagsRoot.getSortedUniqueUsedAbiTags();
   AbiTagList AdditionalAbiTags(ReturnTypeAbiTags.size());
   AdditionalAbiTags.erase(
       std::set_difference(ReturnTypeAbiTags.begin(), ReturnTypeAbiTags.end(),
                           UsedAbiTags.begin(), UsedAbiTags.end(),
                           AdditionalAbiTags.begin()),
       AdditionalAbiTags.end());
 
   // Output name with implicit tags and function encoding from temporary buffer.
   mangleNameWithAbiTags(FD, &AdditionalAbiTags);
   Out << FunctionEncodingStream.str().substr(EncodingPositionStart);
 
   // Function encoding could create new substitutions so we have to add
   // temp mangled substitutions to main mangler.
   extendSubstitutions(&FunctionEncodingMangler);
 }
 
 void CXXNameMangler::mangleFunctionEncodingBareType(const FunctionDecl *FD) {
   if (FD->hasAttr<EnableIfAttr>()) {
     FunctionTypeDepthState Saved = FunctionTypeDepth.push();
     Out << "Ua9enable_ifI";
     for (AttrVec::const_iterator I = FD->getAttrs().begin(),
                                  E = FD->getAttrs().end();
          I != E; ++I) {
       EnableIfAttr *EIA = dyn_cast<EnableIfAttr>(*I);
       if (!EIA)
         continue;
       if (Context.getASTContext().getLangOpts().getClangABICompat() >
           LangOptions::ClangABI::Ver11) {
         mangleTemplateArgExpr(EIA->getCond());
       } else {
         // Prior to Clang 12, we hardcoded the X/E around enable-if's argument,
         // even though <template-arg> should not include an X/E around
         // <expr-primary>.
         Out << 'X';
         mangleExpression(EIA->getCond());
         Out << 'E';
       }
     }
     Out << 'E';
     FunctionTypeDepth.pop(Saved);
   }
 
   // When mangling an inheriting constructor, the bare function type used is
   // that of the inherited constructor.
   if (auto *CD = dyn_cast<CXXConstructorDecl>(FD))
     if (auto Inherited = CD->getInheritedConstructor())
       FD = Inherited.getConstructor();
 
   // Whether the mangling of a function type includes the return type depends on
   // the context and the nature of the function. The rules for deciding whether
   // the return type is included are:
   //
   //   1. Template functions (names or types) have return types encoded, with
   //   the exceptions listed below.
   //   2. Function types not appearing as part of a function name mangling,
   //   e.g. parameters, pointer types, etc., have return type encoded, with the
   //   exceptions listed below.
   //   3. Non-template function names do not have return types encoded.
   //
   // The exceptions mentioned in (1) and (2) above, for which the return type is
   // never included, are
   //   1. Constructors.
   //   2. Destructors.
   //   3. Conversion operator functions, e.g. operator int.
   bool MangleReturnType = false;
   if (FunctionTemplateDecl *PrimaryTemplate = FD->getPrimaryTemplate()) {
     if (!(isa<CXXConstructorDecl>(FD) || isa<CXXDestructorDecl>(FD) ||
           isa<CXXConversionDecl>(FD)))
       MangleReturnType = true;
 
     // Mangle the type of the primary template.
     FD = PrimaryTemplate->getTemplatedDecl();
   }
 
   mangleBareFunctionType(FD->getType()->castAs<FunctionProtoType>(),
                          MangleReturnType, FD);
 }
 
-static const DeclContext *IgnoreLinkageSpecDecls(const DeclContext *DC) {
-  while (isa<LinkageSpecDecl>(DC)) {
-    DC = getEffectiveParentContext(DC);
-  }
-
-  return DC;
-}
-
 /// Return whether a given namespace is the 'std' namespace.
-static bool isStd(const NamespaceDecl *NS) {
-  if (!IgnoreLinkageSpecDecls(getEffectiveParentContext(NS))
-                                ->isTranslationUnit())
+bool CXXNameMangler::isStd(const NamespaceDecl *NS) {
+  if (!Context.getEffectiveParentContext(NS)->isTranslationUnit())
     return false;
 
   const IdentifierInfo *II = NS->getOriginalNamespace()->getIdentifier();
   return II && II->isStr("std");
 }
 
 // isStdNamespace - Return whether a given decl context is a toplevel 'std'
 // namespace.
-static bool isStdNamespace(const DeclContext *DC) {
+bool CXXNameMangler::isStdNamespace(const DeclContext *DC) {
   if (!DC->isNamespace())
     return false;
 
   return isStd(cast<NamespaceDecl>(DC));
 }
 
 static const GlobalDecl
 isTemplate(GlobalDecl GD, const TemplateArgumentList *&TemplateArgs) {
   const NamedDecl *ND = cast<NamedDecl>(GD.getDecl());
   // Check if we have a function template.
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(ND)) {
     if (const TemplateDecl *TD = FD->getPrimaryTemplate()) {
       TemplateArgs = FD->getTemplateSpecializationArgs();
       return GD.getWithDecl(TD);
     }
   }
 
   // Check if we have a class template.
   if (const ClassTemplateSpecializationDecl *Spec =
         dyn_cast<ClassTemplateSpecializationDecl>(ND)) {
     TemplateArgs = &Spec->getTemplateArgs();
     return GD.getWithDecl(Spec->getSpecializedTemplate());
   }
 
   // Check if we have a variable template.
   if (const VarTemplateSpecializationDecl *Spec =
           dyn_cast<VarTemplateSpecializationDecl>(ND)) {
     TemplateArgs = &Spec->getTemplateArgs();
     return GD.getWithDecl(Spec->getSpecializedTemplate());
   }
 
   return GlobalDecl();
 }
 
 static TemplateName asTemplateName(GlobalDecl GD) {
   const TemplateDecl *TD = dyn_cast_or_null<TemplateDecl>(GD.getDecl());
   return TemplateName(const_cast<TemplateDecl*>(TD));
 }
 
 void CXXNameMangler::mangleName(GlobalDecl GD) {
   const NamedDecl *ND = cast<NamedDecl>(GD.getDecl());
   if (const VarDecl *VD = dyn_cast<VarDecl>(ND)) {
     // Variables should have implicit tags from its type.
     AbiTagList VariableTypeAbiTags = makeVariableTypeTags(VD);
     if (VariableTypeAbiTags.empty()) {
       // Simple case no variable type tags.
       mangleNameWithAbiTags(VD, nullptr);
       return;
     }
 
     // Mangle variable name to null stream to collect tags.
     llvm::raw_null_ostream NullOutStream;
     CXXNameMangler VariableNameMangler(*this, NullOutStream);
     VariableNameMangler.disableDerivedAbiTags();
     VariableNameMangler.mangleNameWithAbiTags(VD, nullptr);
 
     // Get tags from variable type that are not present in its name.
     const AbiTagList &UsedAbiTags =
         VariableNameMangler.AbiTagsRoot.getSortedUniqueUsedAbiTags();
     AbiTagList AdditionalAbiTags(VariableTypeAbiTags.size());
     AdditionalAbiTags.erase(
         std::set_difference(VariableTypeAbiTags.begin(),
                             VariableTypeAbiTags.end(), UsedAbiTags.begin(),
                             UsedAbiTags.end(), AdditionalAbiTags.begin()),
         AdditionalAbiTags.end());
 
     // Output name with implicit tags.
     mangleNameWithAbiTags(VD, &AdditionalAbiTags);
   } else {
     mangleNameWithAbiTags(GD, nullptr);
   }
 }
 
+const RecordDecl *CXXNameMangler::GetLocalClassDecl(const Decl *D) {
+  const DeclContext *DC = Context.getEffectiveDeclContext(D);
+  while (!DC->isNamespace() && !DC->isTranslationUnit()) {
+    if (isLocalContainerContext(DC))
+      return dyn_cast<RecordDecl>(D);
+    D = cast<Decl>(DC);
+    DC = Context.getEffectiveDeclContext(D);
+  }
+  return nullptr;
+}
+
 void CXXNameMangler::mangleNameWithAbiTags(GlobalDecl GD,
                                            const AbiTagList *AdditionalAbiTags) {
   const NamedDecl *ND = cast<NamedDecl>(GD.getDecl());
   //  <name> ::= [<module-name>] <nested-name>
   //         ::= [<module-name>] <unscoped-name>
   //         ::= [<module-name>] <unscoped-template-name> <template-args>
   //         ::= <local-name>
   //
-  const DeclContext *DC = getEffectiveDeclContext(ND);
+  const DeclContext *DC = Context.getEffectiveDeclContext(ND);
 
   // If this is an extern variable declared locally, the relevant DeclContext
   // is that of the containing namespace, or the translation unit.
   // FIXME: This is a hack; extern variables declared locally should have
   // a proper semantic declaration context!
   if (isLocalContainerContext(DC) && ND->hasLinkage() && !isLambda(ND))
     while (!DC->isNamespace() && !DC->isTranslationUnit())
-      DC = getEffectiveParentContext(DC);
+      DC = Context.getEffectiveParentContext(DC);
   else if (GetLocalClassDecl(ND)) {
     mangleLocalName(GD, AdditionalAbiTags);
     return;
   }
 
-  DC = IgnoreLinkageSpecDecls(DC);
+  assert(!isa<LinkageSpecDecl>(DC) && "context cannot be LinkageSpecDecl");
 
   if (isLocalContainerContext(DC)) {
     mangleLocalName(GD, AdditionalAbiTags);
     return;
   }
 
   // Do not mangle the owning module for an external linkage declaration.
   // This enables backwards-compatibility with non-modular code, and is
   // a valid choice since conflicts are not permitted by C++ Modules TS
   // [basic.def.odr]/6.2.
   if (!ND->hasExternalFormalLinkage())
     if (Module *M = ND->getOwningModuleForLinkage())
       mangleModuleName(M);
 
   // Closures can require a nested-name mangling even if they're semantically
   // in the global namespace.
   if (const NamedDecl *PrefixND = getClosurePrefix(ND)) {
     mangleNestedNameWithClosurePrefix(GD, PrefixND, AdditionalAbiTags);
     return;
   }
 
   if (DC->isTranslationUnit() || isStdNamespace(DC)) {
     // Check if we have a template.
     const TemplateArgumentList *TemplateArgs = nullptr;
     if (GlobalDecl TD = isTemplate(GD, TemplateArgs)) {
       mangleUnscopedTemplateName(TD, AdditionalAbiTags);
       mangleTemplateArgs(asTemplateName(TD), *TemplateArgs);
       return;
     }
 
     mangleUnscopedName(GD, AdditionalAbiTags);
     return;
   }
 
   mangleNestedName(GD, DC, AdditionalAbiTags);
 }
 
 void CXXNameMangler::mangleModuleName(const Module *M) {
   // Implement the C++ Modules TS name mangling proposal; see
   //     https://gcc.gnu.org/wiki/cxx-modules?action=AttachFile
   //
   //   <module-name> ::= W <unscoped-name>+ E
   //                 ::= W <module-subst> <unscoped-name>* E
   Out << 'W';
   mangleModuleNamePrefix(M->Name);
   Out << 'E';
 }
 
 void CXXNameMangler::mangleModuleNamePrefix(StringRef Name) {
   //  <module-subst> ::= _ <seq-id>          # 0 < seq-id < 10
   //                 ::= W <seq-id - 10> _   # otherwise
   auto It = ModuleSubstitutions.find(Name);
   if (It != ModuleSubstitutions.end()) {
     if (It->second < 10)
       Out << '_' << static_cast<char>('0' + It->second);
     else
       Out << 'W' << (It->second - 10) << '_';
     return;
   }
 
   // FIXME: Preserve hierarchy in module names rather than flattening
   // them to strings; use Module*s as substitution keys.
   auto Parts = Name.rsplit('.');
   if (Parts.second.empty())
     Parts.second = Parts.first;
   else
     mangleModuleNamePrefix(Parts.first);
 
   Out << Parts.second.size() << Parts.second;
   ModuleSubstitutions.insert({Name, ModuleSubstitutions.size()});
 }
 
 void CXXNameMangler::mangleTemplateName(const TemplateDecl *TD,
                                         const TemplateArgument *TemplateArgs,
                                         unsigned NumTemplateArgs) {
-  const DeclContext *DC = IgnoreLinkageSpecDecls(getEffectiveDeclContext(TD));
+  const DeclContext *DC = Context.getEffectiveDeclContext(TD);
 
   if (DC->isTranslationUnit() || isStdNamespace(DC)) {
     mangleUnscopedTemplateName(TD, nullptr);
     mangleTemplateArgs(asTemplateName(TD), TemplateArgs, NumTemplateArgs);
   } else {
     mangleNestedName(TD, TemplateArgs, NumTemplateArgs);
   }
 }
 
 void CXXNameMangler::mangleUnscopedName(GlobalDecl GD,
                                         const AbiTagList *AdditionalAbiTags) {
   const NamedDecl *ND = cast<NamedDecl>(GD.getDecl());
   //  <unscoped-name> ::= <unqualified-name>
   //                  ::= St <unqualified-name>   # ::std::
 
-  if (isStdNamespace(IgnoreLinkageSpecDecls(getEffectiveDeclContext(ND))))
+  if (isStdNamespace(Context.getEffectiveDeclContext(ND)))
     Out << "St";
 
   mangleUnqualifiedName(GD, AdditionalAbiTags);
 }
 
 void CXXNameMangler::mangleUnscopedTemplateName(
     GlobalDecl GD, const AbiTagList *AdditionalAbiTags) {
   const TemplateDecl *ND = cast<TemplateDecl>(GD.getDecl());
   //     <unscoped-template-name> ::= <unscoped-name>
   //                              ::= <substitution>
   if (mangleSubstitution(ND))
     return;
 
   // <template-template-param> ::= <template-param>
   if (const auto *TTP = dyn_cast<TemplateTemplateParmDecl>(ND)) {
     assert(!AdditionalAbiTags &&
            "template template param cannot have abi tags");
     mangleTemplateParameter(TTP->getDepth(), TTP->getIndex());
   } else if (isa<BuiltinTemplateDecl>(ND) || isa<ConceptDecl>(ND)) {
     mangleUnscopedName(GD, AdditionalAbiTags);
   } else {
     mangleUnscopedName(GD.getWithDecl(ND->getTemplatedDecl()), AdditionalAbiTags);
   }
 
   addSubstitution(ND);
 }
 
 void CXXNameMangler::mangleFloat(const llvm::APFloat &f) {
   // ABI:
   //   Floating-point literals are encoded using a fixed-length
   //   lowercase hexadecimal string corresponding to the internal
   //   representation (IEEE on Itanium), high-order bytes first,
   //   without leading zeroes. For example: "Lf bf800000 E" is -1.0f
   //   on Itanium.
   // The 'without leading zeroes' thing seems to be an editorial
   // mistake; see the discussion on cxx-abi-dev beginning on
   // 2012-01-16.
 
   // Our requirements here are just barely weird enough to justify
   // using a custom algorithm instead of post-processing APInt::toString().
 
   llvm::APInt valueBits = f.bitcastToAPInt();
   unsigned numCharacters = (valueBits.getBitWidth() + 3) / 4;
   assert(numCharacters != 0);
 
   // Allocate a buffer of the right number of characters.
   SmallVector<char, 20> buffer(numCharacters);
 
   // Fill the buffer left-to-right.
   for (unsigned stringIndex = 0; stringIndex != numCharacters; ++stringIndex) {
     // The bit-index of the next hex digit.
     unsigned digitBitIndex = 4 * (numCharacters - stringIndex - 1);
 
     // Project out 4 bits starting at 'digitIndex'.
     uint64_t hexDigit = valueBits.getRawData()[digitBitIndex / 64];
     hexDigit >>= (digitBitIndex % 64);
     hexDigit &= 0xF;
 
     // Map that over to a lowercase hex digit.
     static const char charForHex[16] = {
       '0', '1', '2', '3', '4', '5', '6', '7',
       '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'
     };
     buffer[stringIndex] = charForHex[hexDigit];
   }
 
   Out.write(buffer.data(), numCharacters);
 }
 
 void CXXNameMangler::mangleFloatLiteral(QualType T, const llvm::APFloat &V) {
   Out << 'L';
   mangleType(T);
   mangleFloat(V);
   Out << 'E';
 }
 
 void CXXNameMangler::mangleFixedPointLiteral() {
   DiagnosticsEngine &Diags = Context.getDiags();
   unsigned DiagID = Diags.getCustomDiagID(
       DiagnosticsEngine::Error, "cannot mangle fixed point literals yet");
   Diags.Report(DiagID);
 }
 
 void CXXNameMangler::mangleNullPointer(QualType T) {
   //  <expr-primary> ::= L <type> 0 E
   Out << 'L';
   mangleType(T);
   Out << "0E";
 }
 
 void CXXNameMangler::mangleNumber(const llvm::APSInt &Value) {
   if (Value.isSigned() && Value.isNegative()) {
     Out << 'n';
     Value.abs().print(Out, /*signed*/ false);
   } else {
     Value.print(Out, /*signed*/ false);
   }
 }
 
 void CXXNameMangler::mangleNumber(int64_t Number) {
   //  <number> ::= [n] <non-negative decimal integer>
   if (Number < 0) {
     Out << 'n';
     Number = -Number;
   }
 
   Out << Number;
 }
 
 void CXXNameMangler::mangleCallOffset(int64_t NonVirtual, int64_t Virtual) {
   //  <call-offset>  ::= h <nv-offset> _
   //                 ::= v <v-offset> _
   //  <nv-offset>    ::= <offset number>        # non-virtual base override
   //  <v-offset>     ::= <offset number> _ <virtual offset number>
   //                      # virtual base override, with vcall offset
   if (!Virtual) {
     Out << 'h';
     mangleNumber(NonVirtual);
     Out << '_';
     return;
   }
 
   Out << 'v';
   mangleNumber(NonVirtual);
   Out << '_';
   mangleNumber(Virtual);
   Out << '_';
 }
 
 void CXXNameMangler::manglePrefix(QualType type) {
   if (const auto *TST = type->getAs<TemplateSpecializationType>()) {
     if (!mangleSubstitution(QualType(TST, 0))) {
       mangleTemplatePrefix(TST->getTemplateName());
 
       // FIXME: GCC does not appear to mangle the template arguments when
       // the template in question is a dependent template name. Should we
       // emulate that badness?
       mangleTemplateArgs(TST->getTemplateName(), TST->getArgs(),
                          TST->getNumArgs());
       addSubstitution(QualType(TST, 0));
     }
   } else if (const auto *DTST =
                  type->getAs<DependentTemplateSpecializationType>()) {
     if (!mangleSubstitution(QualType(DTST, 0))) {
       TemplateName Template = getASTContext().getDependentTemplateName(
           DTST->getQualifier(), DTST->getIdentifier());
       mangleTemplatePrefix(Template);
 
       // FIXME: GCC does not appear to mangle the template arguments when
       // the template in question is a dependent template name. Should we
       // emulate that badness?
       mangleTemplateArgs(Template, DTST->getArgs(), DTST->getNumArgs());
       addSubstitution(QualType(DTST, 0));
     }
   } else {
     // We use the QualType mangle type variant here because it handles
     // substitutions.
     mangleType(type);
   }
 }
 
 /// Mangle everything prior to the base-unresolved-name in an unresolved-name.
 ///
 /// \param recursive - true if this is being called recursively,
 ///   i.e. if there is more prefix "to the right".
 void CXXNameMangler::mangleUnresolvedPrefix(NestedNameSpecifier *qualifier,
                                             bool recursive) {
 
   // x, ::x
   // <unresolved-name> ::= [gs] <base-unresolved-name>
 
   // T::x / decltype(p)::x
   // <unresolved-name> ::= sr <unresolved-type> <base-unresolved-name>
 
   // T::N::x /decltype(p)::N::x
   // <unresolved-name> ::= srN <unresolved-type> <unresolved-qualifier-level>+ E
   //                       <base-unresolved-name>
 
   // A::x, N::y, A<T>::z; "gs" means leading "::"
   // <unresolved-name> ::= [gs] sr <unresolved-qualifier-level>+ E
   //                       <base-unresolved-name>
 
   switch (qualifier->getKind()) {
   case NestedNameSpecifier::Global:
     Out << "gs";
 
     // We want an 'sr' unless this is the entire NNS.
     if (recursive)
       Out << "sr";
 
     // We never want an 'E' here.
     return;
 
   case NestedNameSpecifier::Super:
     llvm_unreachable("Can't mangle __super specifier");
 
   case NestedNameSpecifier::Namespace:
     if (qualifier->getPrefix())
       mangleUnresolvedPrefix(qualifier->getPrefix(),
                              /*recursive*/ true);
     else
       Out << "sr";
     mangleSourceNameWithAbiTags(qualifier->getAsNamespace());
     break;
   case NestedNameSpecifier::NamespaceAlias:
     if (qualifier->getPrefix())
       mangleUnresolvedPrefix(qualifier->getPrefix(),
                              /*recursive*/ true);
     else
       Out << "sr";
     mangleSourceNameWithAbiTags(qualifier->getAsNamespaceAlias());
     break;
 
   case NestedNameSpecifier::TypeSpec:
   case NestedNameSpecifier::TypeSpecWithTemplate: {
     const Type *type = qualifier->getAsType();
 
     // We only want to use an unresolved-type encoding if this is one of:
     //   - a decltype
     //   - a template type parameter
     //   - a template template parameter with arguments
     // In all of these cases, we should have no prefix.
     if (qualifier->getPrefix()) {
       mangleUnresolvedPrefix(qualifier->getPrefix(),
                              /*recursive*/ true);
     } else {
       // Otherwise, all the cases want this.
       Out << "sr";
     }
 
     if (mangleUnresolvedTypeOrSimpleId(QualType(type, 0), recursive ? "N" : ""))
       return;
 
     break;
   }
 
   case NestedNameSpecifier::Identifier:
     // Member expressions can have these without prefixes.
     if (qualifier->getPrefix())
       mangleUnresolvedPrefix(qualifier->getPrefix(),
                              /*recursive*/ true);
     else
       Out << "sr";
 
     mangleSourceName(qualifier->getAsIdentifier());
     // An Identifier has no type information, so we can't emit abi tags for it.
     break;
   }
 
   // If this was the innermost part of the NNS, and we fell out to
   // here, append an 'E'.
   if (!recursive)
     Out << 'E';
 }
 
 /// Mangle an unresolved-name, which is generally used for names which
 /// weren't resolved to specific entities.
 void CXXNameMangler::mangleUnresolvedName(
     NestedNameSpecifier *qualifier, DeclarationName name,
     const TemplateArgumentLoc *TemplateArgs, unsigned NumTemplateArgs,
     unsigned knownArity) {
   if (qualifier) mangleUnresolvedPrefix(qualifier);
   switch (name.getNameKind()) {
     // <base-unresolved-name> ::= <simple-id>
     case DeclarationName::Identifier:
       mangleSourceName(name.getAsIdentifierInfo());
       break;
     // <base-unresolved-name> ::= dn <destructor-name>
     case DeclarationName::CXXDestructorName:
       Out << "dn";
       mangleUnresolvedTypeOrSimpleId(name.getCXXNameType());
       break;
     // <base-unresolved-name> ::= on <operator-name>
     case DeclarationName::CXXConversionFunctionName:
     case DeclarationName::CXXLiteralOperatorName:
     case DeclarationName::CXXOperatorName:
       Out << "on";
       mangleOperatorName(name, knownArity);
       break;
     case DeclarationName::CXXConstructorName:
       llvm_unreachable("Can't mangle a constructor name!");
     case DeclarationName::CXXUsingDirective:
       llvm_unreachable("Can't mangle a using directive name!");
     case DeclarationName::CXXDeductionGuideName:
       llvm_unreachable("Can't mangle a deduction guide name!");
     case DeclarationName::ObjCMultiArgSelector:
     case DeclarationName::ObjCOneArgSelector:
     case DeclarationName::ObjCZeroArgSelector:
       llvm_unreachable("Can't mangle Objective-C selector names here!");
   }
 
   // The <simple-id> and on <operator-name> productions end in an optional
   // <template-args>.
   if (TemplateArgs)
     mangleTemplateArgs(TemplateName(), TemplateArgs, NumTemplateArgs);
 }
 
 void CXXNameMangler::mangleUnqualifiedName(GlobalDecl GD,
                                            DeclarationName Name,
                                            unsigned KnownArity,
                                            const AbiTagList *AdditionalAbiTags) {
   const NamedDecl *ND = cast_or_null<NamedDecl>(GD.getDecl());
   unsigned Arity = KnownArity;
   //  <unqualified-name> ::= <operator-name>
   //                     ::= <ctor-dtor-name>
   //                     ::= <source-name>
   switch (Name.getNameKind()) {
   case DeclarationName::Identifier: {
     const IdentifierInfo *II = Name.getAsIdentifierInfo();
 
     // We mangle decomposition declarations as the names of their bindings.
     if (auto *DD = dyn_cast<DecompositionDecl>(ND)) {
       // FIXME: Non-standard mangling for decomposition declarations:
       //
       //  <unqualified-name> ::= DC <source-name>* E
       //
       // These can never be referenced across translation units, so we do
       // not need a cross-vendor mangling for anything other than demanglers.
       // Proposed on cxx-abi-dev on 2016-08-12
       Out << "DC";
       for (auto *BD : DD->bindings())
         mangleSourceName(BD->getDeclName().getAsIdentifierInfo());
       Out << 'E';
       writeAbiTags(ND, AdditionalAbiTags);
       break;
     }
 
     if (auto *GD = dyn_cast<MSGuidDecl>(ND)) {
       // We follow MSVC in mangling GUID declarations as if they were variables
       // with a particular reserved name. Continue the pretense here.
       SmallString<sizeof("_GUID_12345678_1234_1234_1234_1234567890ab")> GUID;
       llvm::raw_svector_ostream GUIDOS(GUID);
       Context.mangleMSGuidDecl(GD, GUIDOS);
       Out << GUID.size() << GUID;
       break;
     }
 
     if (auto *TPO = dyn_cast<TemplateParamObjectDecl>(ND)) {
       // Proposed in https://github.com/itanium-cxx-abi/cxx-abi/issues/63.
       Out << "TA";
       mangleValueInTemplateArg(TPO->getType().getUnqualifiedType(),
                                TPO->getValue(), /*TopLevel=*/true);
       break;
     }
 
     if (II) {
       // Match GCC's naming convention for internal linkage symbols, for
       // symbols that are not actually visible outside of this TU. GCC
       // distinguishes between internal and external linkage symbols in
       // its mangling, to support cases like this that were valid C++ prior
       // to DR426:
       //
       //   void test() { extern void foo(); }
       //   static void foo();
       //
       // Don't bother with the L marker for names in anonymous namespaces; the
       // 12_GLOBAL__N_1 mangling is quite sufficient there, and this better
       // matches GCC anyway, because GCC does not treat anonymous namespaces as
       // implying internal linkage.
-      if (isInternalLinkageDecl(ND))
+      if (Context.isInternalLinkageDecl(ND))
         Out << 'L';
 
       auto *FD = dyn_cast<FunctionDecl>(ND);
       bool IsRegCall = FD &&
                        FD->getType()->castAs<FunctionType>()->getCallConv() ==
                            clang::CC_X86RegCall;
       bool IsDeviceStub =
           FD && FD->hasAttr<CUDAGlobalAttr>() &&
           GD.getKernelReferenceKind() == KernelReferenceKind::Stub;
       if (IsDeviceStub)
         mangleDeviceStubName(II);
       else if (IsRegCall)
         mangleRegCallName(II);
       else
         mangleSourceName(II);
 
       writeAbiTags(ND, AdditionalAbiTags);
       break;
     }
 
     // Otherwise, an anonymous entity.  We must have a declaration.
     assert(ND && "mangling empty name without declaration");
 
     if (const NamespaceDecl *NS = dyn_cast<NamespaceDecl>(ND)) {
       if (NS->isAnonymousNamespace()) {
         // This is how gcc mangles these names.
         Out << "12_GLOBAL__N_1";
         break;
       }
     }
 
     if (const VarDecl *VD = dyn_cast<VarDecl>(ND)) {
       // We must have an anonymous union or struct declaration.
       const RecordDecl *RD = VD->getType()->castAs<RecordType>()->getDecl();
 
       // Itanium C++ ABI 5.1.2:
       //
       //   For the purposes of mangling, the name of an anonymous union is
       //   considered to be the name of the first named data member found by a
       //   pre-order, depth-first, declaration-order walk of the data members of
       //   the anonymous union. If there is no such data member (i.e., if all of
       //   the data members in the union are unnamed), then there is no way for
       //   a program to refer to the anonymous union, and there is therefore no
       //   need to mangle its name.
       assert(RD->isAnonymousStructOrUnion()
              && "Expected anonymous struct or union!");
       const FieldDecl *FD = RD->findFirstNamedDataMember();
 
       // It's actually possible for various reasons for us to get here
       // with an empty anonymous struct / union.  Fortunately, it
       // doesn't really matter what name we generate.
       if (!FD) break;
       assert(FD->getIdentifier() && "Data member name isn't an identifier!");
 
       mangleSourceName(FD->getIdentifier());
       // Not emitting abi tags: internal name anyway.
       break;
     }
 
     // Class extensions have no name as a category, and it's possible
     // for them to be the semantic parent of certain declarations
     // (primarily, tag decls defined within declarations).  Such
     // declarations will always have internal linkage, so the name
     // doesn't really matter, but we shouldn't crash on them.  For
     // safety, just handle all ObjC containers here.
     if (isa<ObjCContainerDecl>(ND))
       break;
 
     // We must have an anonymous struct.
     const TagDecl *TD = cast<TagDecl>(ND);
     if (const TypedefNameDecl *D = TD->getTypedefNameForAnonDecl()) {
       assert(TD->getDeclContext() == D->getDeclContext() &&
              "Typedef should not be in another decl context!");
       assert(D->getDeclName().getAsIdentifierInfo() &&
              "Typedef was not named!");
       mangleSourceName(D->getDeclName().getAsIdentifierInfo());
       assert(!AdditionalAbiTags && "Type cannot have additional abi tags");
       // Explicit abi tags are still possible; take from underlying type, not
       // from typedef.
       writeAbiTags(TD, nullptr);
       break;
     }
 
     // <unnamed-type-name> ::= <closure-type-name>
     //
     // <closure-type-name> ::= Ul <lambda-sig> E [ <nonnegative number> ] _
     // <lambda-sig> ::= <template-param-decl>* <parameter-type>+
     //     # Parameter types or 'v' for 'void'.
     if (const CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>(TD)) {
       llvm::Optional<unsigned> DeviceNumber =
           Context.getDiscriminatorOverride()(Context.getASTContext(), Record);
 
       // If we have a device-number via the discriminator, use that to mangle
       // the lambda, otherwise use the typical lambda-mangling-number. In either
       // case, a '0' should be mangled as a normal unnamed class instead of as a
       // lambda.
       if (Record->isLambda() &&
           ((DeviceNumber && *DeviceNumber > 0) ||
            (!DeviceNumber && Record->getLambdaManglingNumber() > 0))) {
         assert(!AdditionalAbiTags &&
                "Lambda type cannot have additional abi tags");
         mangleLambda(Record);
         break;
       }
     }
 
     if (TD->isExternallyVisible()) {
       unsigned UnnamedMangle = getASTContext().getManglingNumber(TD);
       Out << "Ut";
       if (UnnamedMangle > 1)
         Out << UnnamedMangle - 2;
       Out << '_';
       writeAbiTags(TD, AdditionalAbiTags);
       break;
     }
 
     // Get a unique id for the anonymous struct. If it is not a real output
     // ID doesn't matter so use fake one.
     unsigned AnonStructId = NullOut ? 0 : Context.getAnonymousStructId(TD);
 
     // Mangle it as a source name in the form
     // [n] $_<id>
     // where n is the length of the string.
     SmallString<8> Str;
     Str += "$_";
     Str += llvm::utostr(AnonStructId);
 
     Out << Str.size();
     Out << Str;
     break;
   }
 
   case DeclarationName::ObjCZeroArgSelector:
   case DeclarationName::ObjCOneArgSelector:
   case DeclarationName::ObjCMultiArgSelector:
     llvm_unreachable("Can't mangle Objective-C selector names here!");
 
   case DeclarationName::CXXConstructorName: {
     const CXXRecordDecl *InheritedFrom = nullptr;
     TemplateName InheritedTemplateName;
     const TemplateArgumentList *InheritedTemplateArgs = nullptr;
     if (auto Inherited =
             cast<CXXConstructorDecl>(ND)->getInheritedConstructor()) {
       InheritedFrom = Inherited.getConstructor()->getParent();
       InheritedTemplateName =
           TemplateName(Inherited.getConstructor()->getPrimaryTemplate());
       InheritedTemplateArgs =
           Inherited.getConstructor()->getTemplateSpecializationArgs();
     }
 
     if (ND == Structor)
       // If the named decl is the C++ constructor we're mangling, use the type
       // we were given.
       mangleCXXCtorType(static_cast<CXXCtorType>(StructorType), InheritedFrom);
     else
       // Otherwise, use the complete constructor name. This is relevant if a
       // class with a constructor is declared within a constructor.
       mangleCXXCtorType(Ctor_Complete, InheritedFrom);
 
     // FIXME: The template arguments are part of the enclosing prefix or
     // nested-name, but it's more convenient to mangle them here.
     if (InheritedTemplateArgs)
       mangleTemplateArgs(InheritedTemplateName, *InheritedTemplateArgs);
 
     writeAbiTags(ND, AdditionalAbiTags);
     break;
   }
 
   case DeclarationName::CXXDestructorName:
     if (ND == Structor)
       // If the named decl is the C++ destructor we're mangling, use the type we
       // were given.
       mangleCXXDtorType(static_cast<CXXDtorType>(StructorType));
     else
       // Otherwise, use the complete destructor name. This is relevant if a
       // class with a destructor is declared within a destructor.
       mangleCXXDtorType(Dtor_Complete);
     writeAbiTags(ND, AdditionalAbiTags);
     break;
 
   case DeclarationName::CXXOperatorName:
     if (ND && Arity == UnknownArity) {
       Arity = cast<FunctionDecl>(ND)->getNumParams();
 
       // If we have a member function, we need to include the 'this' pointer.
       if (const auto *MD = dyn_cast<CXXMethodDecl>(ND))
         if (!MD->isStatic())
           Arity++;
     }
     LLVM_FALLTHROUGH;
   case DeclarationName::CXXConversionFunctionName:
   case DeclarationName::CXXLiteralOperatorName:
     mangleOperatorName(Name, Arity);
     writeAbiTags(ND, AdditionalAbiTags);
     break;
 
   case DeclarationName::CXXDeductionGuideName:
     llvm_unreachable("Can't mangle a deduction guide name!");
 
   case DeclarationName::CXXUsingDirective:
     llvm_unreachable("Can't mangle a using directive name!");
   }
 }
 
 void CXXNameMangler::mangleRegCallName(const IdentifierInfo *II) {
   // <source-name> ::= <positive length number> __regcall3__ <identifier>
   // <number> ::= [n] <non-negative decimal integer>
   // <identifier> ::= <unqualified source code identifier>
   Out << II->getLength() + sizeof("__regcall3__") - 1 << "__regcall3__"
       << II->getName();
 }
 
 void CXXNameMangler::mangleDeviceStubName(const IdentifierInfo *II) {
   // <source-name> ::= <positive length number> __device_stub__ <identifier>
   // <number> ::= [n] <non-negative decimal integer>
   // <identifier> ::= <unqualified source code identifier>
   Out << II->getLength() + sizeof("__device_stub__") - 1 << "__device_stub__"
       << II->getName();
 }
 
 void CXXNameMangler::mangleSourceName(const IdentifierInfo *II) {
   // <source-name> ::= <positive length number> <identifier>
   // <number> ::= [n] <non-negative decimal integer>
   // <identifier> ::= <unqualified source code identifier>
   Out << II->getLength() << II->getName();
 }
 
 void CXXNameMangler::mangleNestedName(GlobalDecl GD,
                                       const DeclContext *DC,
                                       const AbiTagList *AdditionalAbiTags,
                                       bool NoFunction) {
   const NamedDecl *ND = cast<NamedDecl>(GD.getDecl());
   // <nested-name>
   //   ::= N [<CV-qualifiers>] [<ref-qualifier>] <prefix> <unqualified-name> E
   //   ::= N [<CV-qualifiers>] [<ref-qualifier>] <template-prefix>
   //       <template-args> E
 
   Out << 'N';
   if (const CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(ND)) {
     Qualifiers MethodQuals = Method->getMethodQualifiers();
     // We do not consider restrict a distinguishing attribute for overloading
     // purposes so we must not mangle it.
     MethodQuals.removeRestrict();
     mangleQualifiers(MethodQuals);
     mangleRefQualifier(Method->getRefQualifier());
   }
 
   // Check if we have a template.
   const TemplateArgumentList *TemplateArgs = nullptr;
   if (GlobalDecl TD = isTemplate(GD, TemplateArgs)) {
     mangleTemplatePrefix(TD, NoFunction);
     mangleTemplateArgs(asTemplateName(TD), *TemplateArgs);
   } else {
     manglePrefix(DC, NoFunction);
     mangleUnqualifiedName(GD, AdditionalAbiTags);
   }
 
   Out << 'E';
 }
 void CXXNameMangler::mangleNestedName(const TemplateDecl *TD,
                                       const TemplateArgument *TemplateArgs,
                                       unsigned NumTemplateArgs) {
   // <nested-name> ::= N [<CV-qualifiers>] <template-prefix> <template-args> E
 
   Out << 'N';
 
   mangleTemplatePrefix(TD);
   mangleTemplateArgs(asTemplateName(TD), TemplateArgs, NumTemplateArgs);
 
   Out << 'E';
 }
 
 void CXXNameMangler::mangleNestedNameWithClosurePrefix(
     GlobalDecl GD, const NamedDecl *PrefixND,
     const AbiTagList *AdditionalAbiTags) {
   // A <closure-prefix> represents a variable or field, not a regular
   // DeclContext, so needs special handling. In this case we're mangling a
   // limited form of <nested-name>:
   //
   // <nested-name> ::= N <closure-prefix> <closure-type-name> E
 
   Out << 'N';
 
   mangleClosurePrefix(PrefixND);
   mangleUnqualifiedName(GD, AdditionalAbiTags);
 
   Out << 'E';
 }
 
 static GlobalDecl getParentOfLocalEntity(const DeclContext *DC) {
   GlobalDecl GD;
   // The Itanium spec says:
   // For entities in constructors and destructors, the mangling of the
   // complete object constructor or destructor is used as the base function
   // name, i.e. the C1 or D1 version.
   if (auto *CD = dyn_cast<CXXConstructorDecl>(DC))
     GD = GlobalDecl(CD, Ctor_Complete);
   else if (auto *DD = dyn_cast<CXXDestructorDecl>(DC))
     GD = GlobalDecl(DD, Dtor_Complete);
   else
     GD = GlobalDecl(cast<FunctionDecl>(DC));
   return GD;
 }
 
 void CXXNameMangler::mangleLocalName(GlobalDecl GD,
                                      const AbiTagList *AdditionalAbiTags) {
   const Decl *D = GD.getDecl();
   // <local-name> := Z <function encoding> E <entity name> [<discriminator>]
   //              := Z <function encoding> E s [<discriminator>]
   // <local-name> := Z <function encoding> E d [ <parameter number> ]
   //                 _ <entity name>
   // <discriminator> := _ <non-negative number>
   assert(isa<NamedDecl>(D) || isa<BlockDecl>(D));
   const RecordDecl *RD = GetLocalClassDecl(D);
-  const DeclContext *DC = getEffectiveDeclContext(RD ? RD : D);
+  const DeclContext *DC = Context.getEffectiveDeclContext(RD ? RD : D);
 
   Out << 'Z';
 
   {
     AbiTagState LocalAbiTags(AbiTags);
 
     if (const ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(DC))
       mangleObjCMethodName(MD);
     else if (const BlockDecl *BD = dyn_cast<BlockDecl>(DC))
       mangleBlockForPrefix(BD);
     else
       mangleFunctionEncoding(getParentOfLocalEntity(DC));
 
     // Implicit ABI tags (from namespace) are not available in the following
     // entity; reset to actually emitted tags, which are available.
     LocalAbiTags.setUsedAbiTags(LocalAbiTags.getEmittedAbiTags());
   }
 
   Out << 'E';
 
   // GCC 5.3.0 doesn't emit derived ABI tags for local names but that seems to
   // be a bug that is fixed in trunk.
 
   if (RD) {
     // The parameter number is omitted for the last parameter, 0 for the
     // second-to-last parameter, 1 for the third-to-last parameter, etc. The
     // <entity name> will of course contain a <closure-type-name>: Its
     // numbering will be local to the particular argument in which it appears
     // -- other default arguments do not affect its encoding.
     const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD);
     if (CXXRD && CXXRD->isLambda()) {
       if (const ParmVarDecl *Parm
               = dyn_cast_or_null<ParmVarDecl>(CXXRD->getLambdaContextDecl())) {
         if (const FunctionDecl *Func
               = dyn_cast<FunctionDecl>(Parm->getDeclContext())) {
           Out << 'd';
           unsigned Num = Func->getNumParams() - Parm->getFunctionScopeIndex();
           if (Num > 1)
             mangleNumber(Num - 2);
           Out << '_';
         }
       }
     }
 
     // Mangle the name relative to the closest enclosing function.
     // equality ok because RD derived from ND above
     if (D == RD)  {
       mangleUnqualifiedName(RD, AdditionalAbiTags);
     } else if (const BlockDecl *BD = dyn_cast<BlockDecl>(D)) {
       if (const NamedDecl *PrefixND = getClosurePrefix(BD))
         mangleClosurePrefix(PrefixND, true /*NoFunction*/);
       else
-        manglePrefix(getEffectiveDeclContext(BD), true /*NoFunction*/);
+        manglePrefix(Context.getEffectiveDeclContext(BD), true /*NoFunction*/);
       assert(!AdditionalAbiTags && "Block cannot have additional abi tags");
       mangleUnqualifiedBlock(BD);
     } else {
       const NamedDecl *ND = cast<NamedDecl>(D);
-      mangleNestedName(GD, getEffectiveDeclContext(ND), AdditionalAbiTags,
-                       true /*NoFunction*/);
+      mangleNestedName(GD, Context.getEffectiveDeclContext(ND),
+                       AdditionalAbiTags, true /*NoFunction*/);
     }
   } else if (const BlockDecl *BD = dyn_cast<BlockDecl>(D)) {
     // Mangle a block in a default parameter; see above explanation for
     // lambdas.
     if (const ParmVarDecl *Parm
             = dyn_cast_or_null<ParmVarDecl>(BD->getBlockManglingContextDecl())) {
       if (const FunctionDecl *Func
             = dyn_cast<FunctionDecl>(Parm->getDeclContext())) {
         Out << 'd';
         unsigned Num = Func->getNumParams() - Parm->getFunctionScopeIndex();
         if (Num > 1)
           mangleNumber(Num - 2);
         Out << '_';
       }
     }
 
     assert(!AdditionalAbiTags && "Block cannot have additional abi tags");
     mangleUnqualifiedBlock(BD);
   } else {
     mangleUnqualifiedName(GD, AdditionalAbiTags);
   }
 
   if (const NamedDecl *ND = dyn_cast<NamedDecl>(RD ? RD : D)) {
     unsigned disc;
     if (Context.getNextDiscriminator(ND, disc)) {
       if (disc < 10)
         Out << '_' << disc;
       else
         Out << "__" << disc << '_';
     }
   }
 }
 
 void CXXNameMangler::mangleBlockForPrefix(const BlockDecl *Block) {
   if (GetLocalClassDecl(Block)) {
     mangleLocalName(Block, /* AdditionalAbiTags */ nullptr);
     return;
   }
-  const DeclContext *DC = getEffectiveDeclContext(Block);
+  const DeclContext *DC = Context.getEffectiveDeclContext(Block);
   if (isLocalContainerContext(DC)) {
     mangleLocalName(Block, /* AdditionalAbiTags */ nullptr);
     return;
   }
   if (const NamedDecl *PrefixND = getClosurePrefix(Block))
     mangleClosurePrefix(PrefixND);
   else
     manglePrefix(DC);
   mangleUnqualifiedBlock(Block);
 }
 
 void CXXNameMangler::mangleUnqualifiedBlock(const BlockDecl *Block) {
   // When trying to be ABI-compatibility with clang 12 and before, mangle a
   // <data-member-prefix> now, with no substitutions and no <template-args>.
   if (Decl *Context = Block->getBlockManglingContextDecl()) {
     if (getASTContext().getLangOpts().getClangABICompat() <=
             LangOptions::ClangABI::Ver12 &&
         (isa<VarDecl>(Context) || isa<FieldDecl>(Context)) &&
         Context->getDeclContext()->isRecord()) {
       const auto *ND = cast<NamedDecl>(Context);
       if (ND->getIdentifier()) {
         mangleSourceNameWithAbiTags(ND);
         Out << 'M';
       }
     }
   }
 
   // If we have a block mangling number, use it.
   unsigned Number = Block->getBlockManglingNumber();
   // Otherwise, just make up a number. It doesn't matter what it is because
   // the symbol in question isn't externally visible.
   if (!Number)
     Number = Context.getBlockId(Block, false);
   else {
     // Stored mangling numbers are 1-based.
     --Number;
   }
   Out << "Ub";
   if (Number > 0)
     Out << Number - 1;
   Out << '_';
 }
 
 // <template-param-decl>
 //   ::= Ty                              # template type parameter
 //   ::= Tn <type>                       # template non-type parameter
 //   ::= Tt <template-param-decl>* E     # template template parameter
 //   ::= Tp <template-param-decl>        # template parameter pack
 void CXXNameMangler::mangleTemplateParamDecl(const NamedDecl *Decl) {
   if (auto *Ty = dyn_cast<TemplateTypeParmDecl>(Decl)) {
     if (Ty->isParameterPack())
       Out << "Tp";
     Out << "Ty";
   } else if (auto *Tn = dyn_cast<NonTypeTemplateParmDecl>(Decl)) {
     if (Tn->isExpandedParameterPack()) {
       for (unsigned I = 0, N = Tn->getNumExpansionTypes(); I != N; ++I) {
         Out << "Tn";
         mangleType(Tn->getExpansionType(I));
       }
     } else {
       QualType T = Tn->getType();
       if (Tn->isParameterPack()) {
         Out << "Tp";
         if (auto *PackExpansion = T->getAs<PackExpansionType>())
           T = PackExpansion->getPattern();
       }
       Out << "Tn";
       mangleType(T);
     }
   } else if (auto *Tt = dyn_cast<TemplateTemplateParmDecl>(Decl)) {
     if (Tt->isExpandedParameterPack()) {
       for (unsigned I = 0, N = Tt->getNumExpansionTemplateParameters(); I != N;
            ++I) {
         Out << "Tt";
         for (auto *Param : *Tt->getExpansionTemplateParameters(I))
           mangleTemplateParamDecl(Param);
         Out << "E";
       }
     } else {
       if (Tt->isParameterPack())
         Out << "Tp";
       Out << "Tt";
       for (auto *Param : *Tt->getTemplateParameters())
         mangleTemplateParamDecl(Param);
       Out << "E";
     }
   }
 }
 
 void CXXNameMangler::mangleLambda(const CXXRecordDecl *Lambda) {
   // When trying to be ABI-compatibility with clang 12 and before, mangle a
   // <data-member-prefix> now, with no substitutions.
   if (Decl *Context = Lambda->getLambdaContextDecl()) {
     if (getASTContext().getLangOpts().getClangABICompat() <=
             LangOptions::ClangABI::Ver12 &&
         (isa<VarDecl>(Context) || isa<FieldDecl>(Context)) &&
         !isa<ParmVarDecl>(Context)) {
       if (const IdentifierInfo *Name
             = cast<NamedDecl>(Context)->getIdentifier()) {
         mangleSourceName(Name);
         const TemplateArgumentList *TemplateArgs = nullptr;
         if (GlobalDecl TD = isTemplate(cast<NamedDecl>(Context), TemplateArgs))
           mangleTemplateArgs(asTemplateName(TD), *TemplateArgs);
         Out << 'M';
       }
     }
   }
 
   Out << "Ul";
   mangleLambdaSig(Lambda);
   Out << "E";
 
   // The number is omitted for the first closure type with a given
   // <lambda-sig> in a given context; it is n-2 for the nth closure type
   // (in lexical order) with that same <lambda-sig> and context.
   //
   // The AST keeps track of the number for us.
   //
   // In CUDA/HIP, to ensure the consistent lamba numbering between the device-
   // and host-side compilations, an extra device mangle context may be created
   // if the host-side CXX ABI has different numbering for lambda. In such case,
   // if the mangle context is that device-side one, use the device-side lambda
   // mangling number for this lambda.
   llvm::Optional<unsigned> DeviceNumber =
       Context.getDiscriminatorOverride()(Context.getASTContext(), Lambda);
   unsigned Number =
       DeviceNumber ? *DeviceNumber : Lambda->getLambdaManglingNumber();
 
   assert(Number > 0 && "Lambda should be mangled as an unnamed class");
   if (Number > 1)
     mangleNumber(Number - 2);
   Out << '_';
 }
 
 void CXXNameMangler::mangleLambdaSig(const CXXRecordDecl *Lambda) {
   for (auto *D : Lambda->getLambdaExplicitTemplateParameters())
     mangleTemplateParamDecl(D);
   auto *Proto =
       Lambda->getLambdaTypeInfo()->getType()->castAs<FunctionProtoType>();
   mangleBareFunctionType(Proto, /*MangleReturnType=*/false,
                          Lambda->getLambdaStaticInvoker());
 }
 
 void CXXNameMangler::manglePrefix(NestedNameSpecifier *qualifier) {
   switch (qualifier->getKind()) {
   case NestedNameSpecifier::Global:
     // nothing
     return;
 
   case NestedNameSpecifier::Super:
     llvm_unreachable("Can't mangle __super specifier");
 
   case NestedNameSpecifier::Namespace:
     mangleName(qualifier->getAsNamespace());
     return;
 
   case NestedNameSpecifier::NamespaceAlias:
     mangleName(qualifier->getAsNamespaceAlias()->getNamespace());
     return;
 
   case NestedNameSpecifier::TypeSpec:
   case NestedNameSpecifier::TypeSpecWithTemplate:
     manglePrefix(QualType(qualifier->getAsType(), 0));
     return;
 
   case NestedNameSpecifier::Identifier:
     // Member expressions can have these without prefixes, but that
     // should end up in mangleUnresolvedPrefix instead.
     assert(qualifier->getPrefix());
     manglePrefix(qualifier->getPrefix());
 
     mangleSourceName(qualifier->getAsIdentifier());
     return;
   }
 
   llvm_unreachable("unexpected nested name specifier");
 }
 
 void CXXNameMangler::manglePrefix(const DeclContext *DC, bool NoFunction) {
   //  <prefix> ::= <prefix> <unqualified-name>
   //           ::= <template-prefix> <template-args>
   //           ::= <closure-prefix>
   //           ::= <template-param>
   //           ::= # empty
   //           ::= <substitution>
 
-  DC = IgnoreLinkageSpecDecls(DC);
+  assert(!isa<LinkageSpecDecl>(DC) && "prefix cannot be LinkageSpecDecl");
 
   if (DC->isTranslationUnit())
     return;
 
   if (NoFunction && isLocalContainerContext(DC))
     return;
 
   assert(!isLocalContainerContext(DC));
 
   const NamedDecl *ND = cast<NamedDecl>(DC);
   if (mangleSubstitution(ND))
     return;
 
   // Check if we have a template-prefix or a closure-prefix.
   const TemplateArgumentList *TemplateArgs = nullptr;
   if (GlobalDecl TD = isTemplate(ND, TemplateArgs)) {
     mangleTemplatePrefix(TD);
     mangleTemplateArgs(asTemplateName(TD), *TemplateArgs);
   } else if (const NamedDecl *PrefixND = getClosurePrefix(ND)) {
     mangleClosurePrefix(PrefixND, NoFunction);
     mangleUnqualifiedName(ND, nullptr);
   } else {
-    manglePrefix(getEffectiveDeclContext(ND), NoFunction);
+    manglePrefix(Context.getEffectiveDeclContext(ND), NoFunction);
     mangleUnqualifiedName(ND, nullptr);
   }
 
   addSubstitution(ND);
 }
 
 void CXXNameMangler::mangleTemplatePrefix(TemplateName Template) {
   // <template-prefix> ::= <prefix> <template unqualified-name>
   //                   ::= <template-param>
   //                   ::= <substitution>
   if (TemplateDecl *TD = Template.getAsTemplateDecl())
     return mangleTemplatePrefix(TD);
 
   DependentTemplateName *Dependent = Template.getAsDependentTemplateName();
   assert(Dependent && "unexpected template name kind");
 
   // Clang 11 and before mangled the substitution for a dependent template name
   // after already having emitted (a substitution for) the prefix.
   bool Clang11Compat = getASTContext().getLangOpts().getClangABICompat() <=
                        LangOptions::ClangABI::Ver11;
   if (!Clang11Compat && mangleSubstitution(Template))
     return;
 
   if (NestedNameSpecifier *Qualifier = Dependent->getQualifier())
     manglePrefix(Qualifier);
 
   if (Clang11Compat && mangleSubstitution(Template))
     return;
 
   if (const IdentifierInfo *Id = Dependent->getIdentifier())
     mangleSourceName(Id);
   else
     mangleOperatorName(Dependent->getOperator(), UnknownArity);
 
   addSubstitution(Template);
 }
 
 void CXXNameMangler::mangleTemplatePrefix(GlobalDecl GD,
                                           bool NoFunction) {
   const TemplateDecl *ND = cast<TemplateDecl>(GD.getDecl());
   // <template-prefix> ::= <prefix> <template unqualified-name>
   //                   ::= <template-param>
   //                   ::= <substitution>
   // <template-template-param> ::= <template-param>
   //                               <substitution>
 
   if (mangleSubstitution(ND))
     return;
 
   // <template-template-param> ::= <template-param>
   if (const auto *TTP = dyn_cast<TemplateTemplateParmDecl>(ND)) {
     mangleTemplateParameter(TTP->getDepth(), TTP->getIndex());
   } else {
-    manglePrefix(getEffectiveDeclContext(ND), NoFunction);
+    manglePrefix(Context.getEffectiveDeclContext(ND), NoFunction);
     if (isa<BuiltinTemplateDecl>(ND) || isa<ConceptDecl>(ND))
       mangleUnqualifiedName(GD, nullptr);
     else
       mangleUnqualifiedName(GD.getWithDecl(ND->getTemplatedDecl()), nullptr);
   }
 
   addSubstitution(ND);
 }
 
 const NamedDecl *CXXNameMangler::getClosurePrefix(const Decl *ND) {
   if (getASTContext().getLangOpts().getClangABICompat() <=
       LangOptions::ClangABI::Ver12)
     return nullptr;
 
   const NamedDecl *Context = nullptr;
   if (auto *Block = dyn_cast<BlockDecl>(ND)) {
     Context = dyn_cast_or_null<NamedDecl>(Block->getBlockManglingContextDecl());
   } else if (auto *RD = dyn_cast<CXXRecordDecl>(ND)) {
     if (RD->isLambda())
       Context = dyn_cast_or_null<NamedDecl>(RD->getLambdaContextDecl());
   }
   if (!Context)
     return nullptr;
 
   // Only lambdas within the initializer of a non-local variable or non-static
   // data member get a <closure-prefix>.
   if ((isa<VarDecl>(Context) && cast<VarDecl>(Context)->hasGlobalStorage()) ||
       isa<FieldDecl>(Context))
     return Context;
 
   return nullptr;
 }
 
 void CXXNameMangler::mangleClosurePrefix(const NamedDecl *ND, bool NoFunction) {
   //  <closure-prefix> ::= [ <prefix> ] <unqualified-name> M
   //                   ::= <template-prefix> <template-args> M
   if (mangleSubstitution(ND))
     return;
 
   const TemplateArgumentList *TemplateArgs = nullptr;
   if (GlobalDecl TD = isTemplate(ND, TemplateArgs)) {
     mangleTemplatePrefix(TD, NoFunction);
     mangleTemplateArgs(asTemplateName(TD), *TemplateArgs);
   } else {
-    manglePrefix(getEffectiveDeclContext(ND), NoFunction);
+    manglePrefix(Context.getEffectiveDeclContext(ND), NoFunction);
     mangleUnqualifiedName(ND, nullptr);
   }
 
   Out << 'M';
 
   addSubstitution(ND);
 }
 
 /// Mangles a template name under the production <type>.  Required for
 /// template template arguments.
 ///   <type> ::= <class-enum-type>
 ///          ::= <template-param>
 ///          ::= <substitution>
 void CXXNameMangler::mangleType(TemplateName TN) {
   if (mangleSubstitution(TN))
     return;
 
   TemplateDecl *TD = nullptr;
 
   switch (TN.getKind()) {
   case TemplateName::QualifiedTemplate:
     TD = TN.getAsQualifiedTemplateName()->getTemplateDecl();
     goto HaveDecl;
 
   case TemplateName::Template:
     TD = TN.getAsTemplateDecl();
     goto HaveDecl;
 
   HaveDecl:
     if (auto *TTP = dyn_cast<TemplateTemplateParmDecl>(TD))
       mangleTemplateParameter(TTP->getDepth(), TTP->getIndex());
     else
       mangleName(TD);
     break;
 
   case TemplateName::OverloadedTemplate:
   case TemplateName::AssumedTemplate:
     llvm_unreachable("can't mangle an overloaded template name as a <type>");
 
   case TemplateName::DependentTemplate: {
     const DependentTemplateName *Dependent = TN.getAsDependentTemplateName();
     assert(Dependent->isIdentifier());
 
     // <class-enum-type> ::= <name>
     // <name> ::= <nested-name>
     mangleUnresolvedPrefix(Dependent->getQualifier());
     mangleSourceName(Dependent->getIdentifier());
     break;
   }
 
   case TemplateName::SubstTemplateTemplateParm: {
     // Substituted template parameters are mangled as the substituted
     // template.  This will check for the substitution twice, which is
     // fine, but we have to return early so that we don't try to *add*
     // the substitution twice.
     SubstTemplateTemplateParmStorage *subst
       = TN.getAsSubstTemplateTemplateParm();
     mangleType(subst->getReplacement());
     return;
   }
 
   case TemplateName::SubstTemplateTemplateParmPack: {
     // FIXME: not clear how to mangle this!
     // template <template <class> class T...> class A {
     //   template <template <class> class U...> void foo(B<T,U> x...);
     // };
     Out << "_SUBSTPACK_";
     break;
   }
   }
 
   addSubstitution(TN);
 }
 
 bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty,
                                                     StringRef Prefix) {
   // Only certain other types are valid as prefixes;  enumerate them.
   switch (Ty->getTypeClass()) {
   case Type::Builtin:
   case Type::Complex:
   case Type::Adjusted:
   case Type::Decayed:
   case Type::Pointer:
   case Type::BlockPointer:
   case Type::LValueReference:
   case Type::RValueReference:
   case Type::MemberPointer:
   case Type::ConstantArray:
   case Type::IncompleteArray:
   case Type::VariableArray:
   case Type::DependentSizedArray:
   case Type::DependentAddressSpace:
   case Type::DependentVector:
   case Type::DependentSizedExtVector:
   case Type::Vector:
   case Type::ExtVector:
   case Type::ConstantMatrix:
   case Type::DependentSizedMatrix:
   case Type::FunctionProto:
   case Type::FunctionNoProto:
   case Type::Paren:
   case Type::Attributed:
   case Type::Auto:
   case Type::DeducedTemplateSpecialization:
   case Type::PackExpansion:
   case Type::ObjCObject:
   case Type::ObjCInterface:
   case Type::ObjCObjectPointer:
   case Type::ObjCTypeParam:
   case Type::Atomic:
   case Type::Pipe:
   case Type::MacroQualified:
   case Type::BitInt:
   case Type::DependentBitInt:
     llvm_unreachable("type is illegal as a nested name specifier");
 
   case Type::SubstTemplateTypeParmPack:
     // FIXME: not clear how to mangle this!
     // template <class T...> class A {
     //   template <class U...> void foo(decltype(T::foo(U())) x...);
     // };
     Out << "_SUBSTPACK_";
     break;
 
   // <unresolved-type> ::= <template-param>
   //                   ::= <decltype>
   //                   ::= <template-template-param> <template-args>
   // (this last is not official yet)
   case Type::TypeOfExpr:
   case Type::TypeOf:
   case Type::Decltype:
   case Type::TemplateTypeParm:
   case Type::UnaryTransform:
   case Type::SubstTemplateTypeParm:
   unresolvedType:
     // Some callers want a prefix before the mangled type.
     Out << Prefix;
 
     // This seems to do everything we want.  It's not really
     // sanctioned for a substituted template parameter, though.
     mangleType(Ty);
 
     // We never want to print 'E' directly after an unresolved-type,
     // so we return directly.
     return true;
 
   case Type::Typedef:
     mangleSourceNameWithAbiTags(cast<TypedefType>(Ty)->getDecl());
     break;
 
   case Type::UnresolvedUsing:
     mangleSourceNameWithAbiTags(
         cast<UnresolvedUsingType>(Ty)->getDecl());
     break;
 
   case Type::Enum:
   case Type::Record:
     mangleSourceNameWithAbiTags(cast<TagType>(Ty)->getDecl());
     break;
 
   case Type::TemplateSpecialization: {
     const TemplateSpecializationType *TST =
         cast<TemplateSpecializationType>(Ty);
     TemplateName TN = TST->getTemplateName();
     switch (TN.getKind()) {
     case TemplateName::Template:
     case TemplateName::QualifiedTemplate: {
       TemplateDecl *TD = TN.getAsTemplateDecl();
 
       // If the base is a template template parameter, this is an
       // unresolved type.
       assert(TD && "no template for template specialization type");
       if (isa<TemplateTemplateParmDecl>(TD))
         goto unresolvedType;
 
       mangleSourceNameWithAbiTags(TD);
       break;
     }
 
     case TemplateName::OverloadedTemplate:
     case TemplateName::AssumedTemplate:
     case TemplateName::DependentTemplate:
       llvm_unreachable("invalid base for a template specialization type");
 
     case TemplateName::SubstTemplateTemplateParm: {
       SubstTemplateTemplateParmStorage *subst =
           TN.getAsSubstTemplateTemplateParm();
       mangleExistingSubstitution(subst->getReplacement());
       break;
     }
 
     case TemplateName::SubstTemplateTemplateParmPack: {
       // FIXME: not clear how to mangle this!
       // template <template <class U> class T...> class A {
       //   template <class U...> void foo(decltype(T<U>::foo) x...);
       // };
       Out << "_SUBSTPACK_";
       break;
     }
     }
 
     // Note: we don't pass in the template name here. We are mangling the
     // original source-level template arguments, so we shouldn't consider
     // conversions to the corresponding template parameter.
     // FIXME: Other compilers mangle partially-resolved template arguments in
     // unresolved-qualifier-levels.
     mangleTemplateArgs(TemplateName(), TST->getArgs(), TST->getNumArgs());
     break;
   }
 
   case Type::InjectedClassName:
     mangleSourceNameWithAbiTags(
         cast<InjectedClassNameType>(Ty)->getDecl());
     break;
 
   case Type::DependentName:
     mangleSourceName(cast<DependentNameType>(Ty)->getIdentifier());
     break;
 
   case Type::DependentTemplateSpecialization: {
     const DependentTemplateSpecializationType *DTST =
         cast<DependentTemplateSpecializationType>(Ty);
     TemplateName Template = getASTContext().getDependentTemplateName(
         DTST->getQualifier(), DTST->getIdentifier());
     mangleSourceName(DTST->getIdentifier());
     mangleTemplateArgs(Template, DTST->getArgs(), DTST->getNumArgs());
     break;
   }
 
   case Type::Using:
     return mangleUnresolvedTypeOrSimpleId(cast<UsingType>(Ty)->desugar(),
                                           Prefix);
   case Type::Elaborated:
     return mangleUnresolvedTypeOrSimpleId(
         cast<ElaboratedType>(Ty)->getNamedType(), Prefix);
   }
 
   return false;
 }
 
 void CXXNameMangler::mangleOperatorName(DeclarationName Name, unsigned Arity) {
   switch (Name.getNameKind()) {
   case DeclarationName::CXXConstructorName:
   case DeclarationName::CXXDestructorName:
   case DeclarationName::CXXDeductionGuideName:
   case DeclarationName::CXXUsingDirective:
   case DeclarationName::Identifier:
   case DeclarationName::ObjCMultiArgSelector:
   case DeclarationName::ObjCOneArgSelector:
   case DeclarationName::ObjCZeroArgSelector:
     llvm_unreachable("Not an operator name");
 
   case DeclarationName::CXXConversionFunctionName:
     // <operator-name> ::= cv <type>    # (cast)
     Out << "cv";
     mangleType(Name.getCXXNameType());
     break;
 
   case DeclarationName::CXXLiteralOperatorName:
     Out << "li";
     mangleSourceName(Name.getCXXLiteralIdentifier());
     return;
 
   case DeclarationName::CXXOperatorName:
     mangleOperatorName(Name.getCXXOverloadedOperator(), Arity);
     break;
   }
 }
 
 void
 CXXNameMangler::mangleOperatorName(OverloadedOperatorKind OO, unsigned Arity) {
   switch (OO) {
   // <operator-name> ::= nw     # new
   case OO_New: Out << "nw"; break;
   //              ::= na        # new[]
   case OO_Array_New: Out << "na"; break;
   //              ::= dl        # delete
   case OO_Delete: Out << "dl"; break;
   //              ::= da        # delete[]
   case OO_Array_Delete: Out << "da"; break;
   //              ::= ps        # + (unary)
   //              ::= pl        # + (binary or unknown)
   case OO_Plus:
     Out << (Arity == 1? "ps" : "pl"); break;
   //              ::= ng        # - (unary)
   //              ::= mi        # - (binary or unknown)
   case OO_Minus:
     Out << (Arity == 1? "ng" : "mi"); break;
   //              ::= ad        # & (unary)
   //              ::= an        # & (binary or unknown)
   case OO_Amp:
     Out << (Arity == 1? "ad" : "an"); break;
   //              ::= de        # * (unary)
   //              ::= ml        # * (binary or unknown)
   case OO_Star:
     // Use binary when unknown.
     Out << (Arity == 1? "de" : "ml"); break;
   //              ::= co        # ~
   case OO_Tilde: Out << "co"; break;
   //              ::= dv        # /
   case OO_Slash: Out << "dv"; break;
   //              ::= rm        # %
   case OO_Percent: Out << "rm"; break;
   //              ::= or        # |
   case OO_Pipe: Out << "or"; break;
   //              ::= eo        # ^
   case OO_Caret: Out << "eo"; break;
   //              ::= aS        # =
   case OO_Equal: Out << "aS"; break;
   //              ::= pL        # +=
   case OO_PlusEqual: Out << "pL"; break;
   //              ::= mI        # -=
   case OO_MinusEqual: Out << "mI"; break;
   //              ::= mL        # *=
   case OO_StarEqual: Out << "mL"; break;
   //              ::= dV        # /=
   case OO_SlashEqual: Out << "dV"; break;
   //              ::= rM        # %=
   case OO_PercentEqual: Out << "rM"; break;
   //              ::= aN        # &=
   case OO_AmpEqual: Out << "aN"; break;
   //              ::= oR        # |=
   case OO_PipeEqual: Out << "oR"; break;
   //              ::= eO        # ^=
   case OO_CaretEqual: Out << "eO"; break;
   //              ::= ls        # <<
   case OO_LessLess: Out << "ls"; break;
   //              ::= rs        # >>
   case OO_GreaterGreater: Out << "rs"; break;
   //              ::= lS        # <<=
   case OO_LessLessEqual: Out << "lS"; break;
   //              ::= rS        # >>=
   case OO_GreaterGreaterEqual: Out << "rS"; break;
   //              ::= eq        # ==
   case OO_EqualEqual: Out << "eq"; break;
   //              ::= ne        # !=
   case OO_ExclaimEqual: Out << "ne"; break;
   //              ::= lt        # <
   case OO_Less: Out << "lt"; break;
   //              ::= gt        # >
   case OO_Greater: Out << "gt"; break;
   //              ::= le        # <=
   case OO_LessEqual: Out << "le"; break;
   //              ::= ge        # >=
   case OO_GreaterEqual: Out << "ge"; break;
   //              ::= nt        # !
   case OO_Exclaim: Out << "nt"; break;
   //              ::= aa        # &&
   case OO_AmpAmp: Out << "aa"; break;
   //              ::= oo        # ||
   case OO_PipePipe: Out << "oo"; break;
   //              ::= pp        # ++
   case OO_PlusPlus: Out << "pp"; break;
   //              ::= mm        # --
   case OO_MinusMinus: Out << "mm"; break;
   //              ::= cm        # ,
   case OO_Comma: Out << "cm"; break;
   //              ::= pm        # ->*
   case OO_ArrowStar: Out << "pm"; break;
   //              ::= pt        # ->
   case OO_Arrow: Out << "pt"; break;
   //              ::= cl        # ()
   case OO_Call: Out << "cl"; break;
   //              ::= ix        # []
   case OO_Subscript: Out << "ix"; break;
 
   //              ::= qu        # ?
   // The conditional operator can't be overloaded, but we still handle it when
   // mangling expressions.
   case OO_Conditional: Out << "qu"; break;
   // Proposal on cxx-abi-dev, 2015-10-21.
   //              ::= aw        # co_await
   case OO_Coawait: Out << "aw"; break;
   // Proposed in cxx-abi github issue 43.
   //              ::= ss        # <=>
   case OO_Spaceship: Out << "ss"; break;
 
   case OO_None:
   case NUM_OVERLOADED_OPERATORS:
     llvm_unreachable("Not an overloaded operator");
   }
 }
 
 void CXXNameMangler::mangleQualifiers(Qualifiers Quals, const DependentAddressSpaceType *DAST) {
   // Vendor qualifiers come first and if they are order-insensitive they must
   // be emitted in reversed alphabetical order, see Itanium ABI 5.1.5.
 
   // <type> ::= U <addrspace-expr>
   if (DAST) {
     Out << "U2ASI";
     mangleExpression(DAST->getAddrSpaceExpr());
     Out << "E";
   }
 
   // Address space qualifiers start with an ordinary letter.
   if (Quals.hasAddressSpace()) {
     // Address space extension:
     //
     //   <type> ::= U <target-addrspace>
     //   <type> ::= U <OpenCL-addrspace>
     //   <type> ::= U <CUDA-addrspace>
 
     SmallString<64> ASString;
     LangAS AS = Quals.getAddressSpace();
 
     if (Context.getASTContext().addressSpaceMapManglingFor(AS)) {
       //  <target-addrspace> ::= "AS" <address-space-number>
       unsigned TargetAS = Context.getASTContext().getTargetAddressSpace(AS);
       if (TargetAS != 0 ||
           Context.getASTContext().getTargetAddressSpace(LangAS::Default) != 0)
         ASString = "AS" + llvm::utostr(TargetAS);
     } else {
       switch (AS) {
       default: llvm_unreachable("Not a language specific address space");
       //  <OpenCL-addrspace> ::= "CL" [ "global" | "local" | "constant" |
       //                                "private"| "generic" | "device" |
       //                                "host" ]
       case LangAS::opencl_global:
         ASString = "CLglobal";
         break;
       case LangAS::opencl_global_device:
         ASString = "CLdevice";
         break;
       case LangAS::opencl_global_host:
         ASString = "CLhost";
         break;
       case LangAS::opencl_local:
         ASString = "CLlocal";
         break;
       case LangAS::opencl_constant:
         ASString = "CLconstant";
         break;
       case LangAS::opencl_private:
         ASString = "CLprivate";
         break;
       case LangAS::opencl_generic:
         ASString = "CLgeneric";
         break;
       //  <SYCL-addrspace> ::= "SY" [ "global" | "local" | "private" |
       //                              "device" | "host" ]
       case LangAS::sycl_global:
         ASString = "SYglobal";
         break;
       case LangAS::sycl_global_device:
         ASString = "SYdevice";
         break;
       case LangAS::sycl_global_host:
         ASString = "SYhost";
         break;
       case LangAS::sycl_local:
         ASString = "SYlocal";
         break;
       case LangAS::sycl_private:
         ASString = "SYprivate";
         break;
       //  <CUDA-addrspace> ::= "CU" [ "device" | "constant" | "shared" ]
       case LangAS::cuda_device:
         ASString = "CUdevice";
         break;
       case LangAS::cuda_constant:
         ASString = "CUconstant";
         break;
       case LangAS::cuda_shared:
         ASString = "CUshared";
         break;
       //  <ptrsize-addrspace> ::= [ "ptr32_sptr" | "ptr32_uptr" | "ptr64" ]
       case LangAS::ptr32_sptr:
         ASString = "ptr32_sptr";
         break;
       case LangAS::ptr32_uptr:
         ASString = "ptr32_uptr";
         break;
       case LangAS::ptr64:
         ASString = "ptr64";
         break;
       }
     }
     if (!ASString.empty())
       mangleVendorQualifier(ASString);
   }
 
   // The ARC ownership qualifiers start with underscores.
   // Objective-C ARC Extension:
   //
   //   <type> ::= U "__strong"
   //   <type> ::= U "__weak"
   //   <type> ::= U "__autoreleasing"
   //
   // Note: we emit __weak first to preserve the order as
   // required by the Itanium ABI.
   if (Quals.getObjCLifetime() == Qualifiers::OCL_Weak)
     mangleVendorQualifier("__weak");
 
   // __unaligned (from -fms-extensions)
   if (Quals.hasUnaligned())
     mangleVendorQualifier("__unaligned");
 
   // Remaining ARC ownership qualifiers.
   switch (Quals.getObjCLifetime()) {
   case Qualifiers::OCL_None:
     break;
 
   case Qualifiers::OCL_Weak:
     // Do nothing as we already handled this case above.
     break;
 
   case Qualifiers::OCL_Strong:
     mangleVendorQualifier("__strong");
     break;
 
   case Qualifiers::OCL_Autoreleasing:
     mangleVendorQualifier("__autoreleasing");
     break;
 
   case Qualifiers::OCL_ExplicitNone:
     // The __unsafe_unretained qualifier is *not* mangled, so that
     // __unsafe_unretained types in ARC produce the same manglings as the
     // equivalent (but, naturally, unqualified) types in non-ARC, providing
     // better ABI compatibility.
     //
     // It's safe to do this because unqualified 'id' won't show up
     // in any type signatures that need to be mangled.
     break;
   }
 
   // <CV-qualifiers> ::= [r] [V] [K]    # restrict (C99), volatile, const
   if (Quals.hasRestrict())
     Out << 'r';
   if (Quals.hasVolatile())
     Out << 'V';
   if (Quals.hasConst())
     Out << 'K';
 }
 
 void CXXNameMangler::mangleVendorQualifier(StringRef name) {
   Out << 'U' << name.size() << name;
 }
 
 void CXXNameMangler::mangleRefQualifier(RefQualifierKind RefQualifier) {
   // <ref-qualifier> ::= R                # lvalue reference
   //                 ::= O                # rvalue-reference
   switch (RefQualifier) {
   case RQ_None:
     break;
 
   case RQ_LValue:
     Out << 'R';
     break;
 
   case RQ_RValue:
     Out << 'O';
     break;
   }
 }
 
 void CXXNameMangler::mangleObjCMethodName(const ObjCMethodDecl *MD) {
   Context.mangleObjCMethodNameAsSourceName(MD, Out);
 }
 
 static bool isTypeSubstitutable(Qualifiers Quals, const Type *Ty,
                                 ASTContext &Ctx) {
   if (Quals)
     return true;
   if (Ty->isSpecificBuiltinType(BuiltinType::ObjCSel))
     return true;
   if (Ty->isOpenCLSpecificType())
     return true;
   if (Ty->isBuiltinType())
     return false;
   // Through to Clang 6.0, we accidentally treated undeduced auto types as
   // substitution candidates.
   if (Ctx.getLangOpts().getClangABICompat() > LangOptions::ClangABI::Ver6 &&
       isa<AutoType>(Ty))
     return false;
   // A placeholder type for class template deduction is substitutable with
   // its corresponding template name; this is handled specially when mangling
   // the type.
   if (auto *DeducedTST = Ty->getAs<DeducedTemplateSpecializationType>())
     if (DeducedTST->getDeducedType().isNull())
       return false;
   return true;
 }
 
 void CXXNameMangler::mangleType(QualType T) {
   // If our type is instantiation-dependent but not dependent, we mangle
   // it as it was written in the source, removing any top-level sugar.
   // Otherwise, use the canonical type.
   //
   // FIXME: This is an approximation of the instantiation-dependent name
   // mangling rules, since we should really be using the type as written and
   // augmented via semantic analysis (i.e., with implicit conversions and
   // default template arguments) for any instantiation-dependent type.
   // Unfortunately, that requires several changes to our AST:
   //   - Instantiation-dependent TemplateSpecializationTypes will need to be
   //     uniqued, so that we can handle substitutions properly
   //   - Default template arguments will need to be represented in the
   //     TemplateSpecializationType, since they need to be mangled even though
   //     they aren't written.
   //   - Conversions on non-type template arguments need to be expressed, since
   //     they can affect the mangling of sizeof/alignof.
   //
   // FIXME: This is wrong when mapping to the canonical type for a dependent
   // type discards instantiation-dependent portions of the type, such as for:
   //
   //   template<typename T, int N> void f(T (&)[sizeof(N)]);
   //   template<typename T> void f(T() throw(typename T::type)); (pre-C++17)
   //
   // It's also wrong in the opposite direction when instantiation-dependent,
   // canonically-equivalent types differ in some irrelevant portion of inner
   // type sugar. In such cases, we fail to form correct substitutions, eg:
   //
   //   template<int N> void f(A<sizeof(N)> *, A<sizeof(N)> (*));
   //
   // We should instead canonicalize the non-instantiation-dependent parts,
   // regardless of whether the type as a whole is dependent or instantiation
   // dependent.
   if (!T->isInstantiationDependentType() || T->isDependentType())
     T = T.getCanonicalType();
   else {
     // Desugar any types that are purely sugar.
     do {
       // Don't desugar through template specialization types that aren't
       // type aliases. We need to mangle the template arguments as written.
       if (const TemplateSpecializationType *TST
                                       = dyn_cast<TemplateSpecializationType>(T))
         if (!TST->isTypeAlias())
           break;
 
       // FIXME: We presumably shouldn't strip off ElaboratedTypes with
       // instantation-dependent qualifiers. See
       // https://github.com/itanium-cxx-abi/cxx-abi/issues/114.
 
       QualType Desugared
         = T.getSingleStepDesugaredType(Context.getASTContext());
       if (Desugared == T)
         break;
 
       T = Desugared;
     } while (true);
   }
   SplitQualType split = T.split();
   Qualifiers quals = split.Quals;
   const Type *ty = split.Ty;
 
   bool isSubstitutable =
     isTypeSubstitutable(quals, ty, Context.getASTContext());
   if (isSubstitutable && mangleSubstitution(T))
     return;
 
   // If we're mangling a qualified array type, push the qualifiers to
   // the element type.
   if (quals && isa<ArrayType>(T)) {
     ty = Context.getASTContext().getAsArrayType(T);
     quals = Qualifiers();
 
     // Note that we don't update T: we want to add the
     // substitution at the original type.
   }
 
   if (quals || ty->isDependentAddressSpaceType()) {
     if (const DependentAddressSpaceType *DAST =
         dyn_cast<DependentAddressSpaceType>(ty)) {
       SplitQualType splitDAST = DAST->getPointeeType().split();
       mangleQualifiers(splitDAST.Quals, DAST);
       mangleType(QualType(splitDAST.Ty, 0));
     } else {
       mangleQualifiers(quals);
 
       // Recurse:  even if the qualified type isn't yet substitutable,
       // the unqualified type might be.
       mangleType(QualType(ty, 0));
     }
   } else {
     switch (ty->getTypeClass()) {
 #define ABSTRACT_TYPE(CLASS, PARENT)
 #define NON_CANONICAL_TYPE(CLASS, PARENT) \
     case Type::CLASS: \
       llvm_unreachable("can't mangle non-canonical type " #CLASS "Type"); \
       return;
 #define TYPE(CLASS, PARENT) \
     case Type::CLASS: \
       mangleType(static_cast<const CLASS##Type*>(ty)); \
       break;
 #include "clang/AST/TypeNodes.inc"
     }
   }
 
   // Add the substitution.
   if (isSubstitutable)
     addSubstitution(T);
 }
 
 void CXXNameMangler::mangleNameOrStandardSubstitution(const NamedDecl *ND) {
   if (!mangleStandardSubstitution(ND))
     mangleName(ND);
 }
 
 void CXXNameMangler::mangleType(const BuiltinType *T) {
   //  <type>         ::= <builtin-type>
   //  <builtin-type> ::= v  # void
   //                 ::= w  # wchar_t
   //                 ::= b  # bool
   //                 ::= c  # char
   //                 ::= a  # signed char
   //                 ::= h  # unsigned char
   //                 ::= s  # short
   //                 ::= t  # unsigned short
   //                 ::= i  # int
   //                 ::= j  # unsigned int
   //                 ::= l  # long
   //                 ::= m  # unsigned long
   //                 ::= x  # long long, __int64
   //                 ::= y  # unsigned long long, __int64
   //                 ::= n  # __int128
   //                 ::= o  # unsigned __int128
   //                 ::= f  # float
   //                 ::= d  # double
   //                 ::= e  # long double, __float80
   //                 ::= g  # __float128
   //                 ::= g  # __ibm128
   // UNSUPPORTED:    ::= Dd # IEEE 754r decimal floating point (64 bits)
   // UNSUPPORTED:    ::= De # IEEE 754r decimal floating point (128 bits)
   // UNSUPPORTED:    ::= Df # IEEE 754r decimal floating point (32 bits)
   //                 ::= Dh # IEEE 754r half-precision floating point (16 bits)
   //                 ::= DF <number> _ # ISO/IEC TS 18661 binary floating point type _FloatN (N bits);
   //                 ::= Di # char32_t
   //                 ::= Ds # char16_t
   //                 ::= Dn # std::nullptr_t (i.e., decltype(nullptr))
   //                 ::= u <source-name>    # vendor extended type
   std::string type_name;
   switch (T->getKind()) {
   case BuiltinType::Void:
     Out << 'v';
     break;
   case BuiltinType::Bool:
     Out << 'b';
     break;
   case BuiltinType::Char_U:
   case BuiltinType::Char_S:
     Out << 'c';
     break;
   case BuiltinType::UChar:
     Out << 'h';
     break;
   case BuiltinType::UShort:
     Out << 't';
     break;
   case BuiltinType::UInt:
     Out << 'j';
     break;
   case BuiltinType::ULong:
     Out << 'm';
     break;
   case BuiltinType::ULongLong:
     Out << 'y';
     break;
   case BuiltinType::UInt128:
     Out << 'o';
     break;
   case BuiltinType::SChar:
     Out << 'a';
     break;
   case BuiltinType::WChar_S:
   case BuiltinType::WChar_U:
     Out << 'w';
     break;
   case BuiltinType::Char8:
     Out << "Du";
     break;
   case BuiltinType::Char16:
     Out << "Ds";
     break;
   case BuiltinType::Char32:
     Out << "Di";
     break;
   case BuiltinType::Short:
     Out << 's';
     break;
   case BuiltinType::Int:
     Out << 'i';
     break;
   case BuiltinType::Long:
     Out << 'l';
     break;
   case BuiltinType::LongLong:
     Out << 'x';
     break;
   case BuiltinType::Int128:
     Out << 'n';
     break;
   case BuiltinType::Float16:
     Out << "DF16_";
     break;
   case BuiltinType::ShortAccum:
   case BuiltinType::Accum:
   case BuiltinType::LongAccum:
   case BuiltinType::UShortAccum:
   case BuiltinType::UAccum:
   case BuiltinType::ULongAccum:
   case BuiltinType::ShortFract:
   case BuiltinType::Fract:
   case BuiltinType::LongFract:
   case BuiltinType::UShortFract:
   case BuiltinType::UFract:
   case BuiltinType::ULongFract:
   case BuiltinType::SatShortAccum:
   case BuiltinType::SatAccum:
   case BuiltinType::SatLongAccum:
   case BuiltinType::SatUShortAccum:
   case BuiltinType::SatUAccum:
   case BuiltinType::SatULongAccum:
   case BuiltinType::SatShortFract:
   case BuiltinType::SatFract:
   case BuiltinType::SatLongFract:
   case BuiltinType::SatUShortFract:
   case BuiltinType::SatUFract:
   case BuiltinType::SatULongFract:
     llvm_unreachable("Fixed point types are disabled for c++");
   case BuiltinType::Half:
     Out << "Dh";
     break;
   case BuiltinType::Float:
     Out << 'f';
     break;
   case BuiltinType::Double:
     Out << 'd';
     break;
   case BuiltinType::LongDouble: {
     const TargetInfo *TI = getASTContext().getLangOpts().OpenMP &&
                                    getASTContext().getLangOpts().OpenMPIsDevice
                                ? getASTContext().getAuxTargetInfo()
                                : &getASTContext().getTargetInfo();
     Out << TI->getLongDoubleMangling();
     break;
   }
   case BuiltinType::Float128: {
     const TargetInfo *TI = getASTContext().getLangOpts().OpenMP &&
                                    getASTContext().getLangOpts().OpenMPIsDevice
                                ? getASTContext().getAuxTargetInfo()
                                : &getASTContext().getTargetInfo();
     Out << TI->getFloat128Mangling();
     break;
   }
   case BuiltinType::BFloat16: {
     const TargetInfo *TI = &getASTContext().getTargetInfo();
     Out << TI->getBFloat16Mangling();
     break;
   }
   case BuiltinType::Ibm128: {
     const TargetInfo *TI = &getASTContext().getTargetInfo();
     Out << TI->getIbm128Mangling();
     break;
   }
   case BuiltinType::NullPtr:
     Out << "Dn";
     break;
 
 #define BUILTIN_TYPE(Id, SingletonId)
 #define PLACEHOLDER_TYPE(Id, SingletonId) \
   case BuiltinType::Id:
 #include "clang/AST/BuiltinTypes.def"
   case BuiltinType::Dependent:
     if (!NullOut)
       llvm_unreachable("mangling a placeholder type");
     break;
   case BuiltinType::ObjCId:
     Out << "11objc_object";
     break;
   case BuiltinType::ObjCClass:
     Out << "10objc_class";
     break;
   case BuiltinType::ObjCSel:
     Out << "13objc_selector";
     break;
 #define IMAGE_TYPE(ImgType, Id, SingletonId, Access, Suffix) \
   case BuiltinType::Id: \
     type_name = "ocl_" #ImgType "_" #Suffix; \
     Out << type_name.size() << type_name; \
     break;
 #include "clang/Basic/OpenCLImageTypes.def"
   case BuiltinType::OCLSampler:
     Out << "11ocl_sampler";
     break;
   case BuiltinType::OCLEvent:
     Out << "9ocl_event";
     break;
   case BuiltinType::OCLClkEvent:
     Out << "12ocl_clkevent";
     break;
   case BuiltinType::OCLQueue:
     Out << "9ocl_queue";
     break;
   case BuiltinType::OCLReserveID:
     Out << "13ocl_reserveid";
     break;
 #define EXT_OPAQUE_TYPE(ExtType, Id, Ext) \
   case BuiltinType::Id: \
     type_name = "ocl_" #ExtType; \
     Out << type_name.size() << type_name; \
     break;
 #include "clang/Basic/OpenCLExtensionTypes.def"
   // The SVE types are effectively target-specific.  The mangling scheme
   // is defined in the appendices to the Procedure Call Standard for the
   // Arm Architecture.
 #define SVE_VECTOR_TYPE(InternalName, MangledName, Id, SingletonId, NumEls,    \
                         ElBits, IsSigned, IsFP, IsBF)                          \
   case BuiltinType::Id:                                                        \
     type_name = MangledName;                                                   \
     Out << (type_name == InternalName ? "u" : "") << type_name.size()          \
         << type_name;                                                          \
     break;
 #define SVE_PREDICATE_TYPE(InternalName, MangledName, Id, SingletonId, NumEls) \
   case BuiltinType::Id:                                                        \
     type_name = MangledName;                                                   \
     Out << (type_name == InternalName ? "u" : "") << type_name.size()          \
         << type_name;                                                          \
     break;
 #include "clang/Basic/AArch64SVEACLETypes.def"
 #define PPC_VECTOR_TYPE(Name, Id, Size) \
   case BuiltinType::Id: \
     type_name = #Name; \
     Out << 'u' << type_name.size() << type_name; \
     break;
 #include "clang/Basic/PPCTypes.def"
     // TODO: Check the mangling scheme for RISC-V V.
 #define RVV_TYPE(Name, Id, SingletonId)                                        \
   case BuiltinType::Id:                                                        \
     type_name = Name;                                                          \
     Out << 'u' << type_name.size() << type_name;                               \
     break;
 #include "clang/Basic/RISCVVTypes.def"
   }
 }
 
 StringRef CXXNameMangler::getCallingConvQualifierName(CallingConv CC) {
   switch (CC) {
   case CC_C:
     return "";
 
   case CC_X86VectorCall:
   case CC_X86Pascal:
   case CC_X86RegCall:
   case CC_AAPCS:
   case CC_AAPCS_VFP:
   case CC_AArch64VectorCall:
   case CC_IntelOclBicc:
   case CC_SpirFunction:
   case CC_OpenCLKernel:
   case CC_PreserveMost:
   case CC_PreserveAll:
     // FIXME: we should be mangling all of the above.
     return "";
 
   case CC_X86ThisCall:
     // FIXME: To match mingw GCC, thiscall should only be mangled in when it is
     // used explicitly. At this point, we don't have that much information in
     // the AST, since clang tends to bake the convention into the canonical
     // function type. thiscall only rarely used explicitly, so don't mangle it
     // for now.
     return "";
 
   case CC_X86StdCall:
     return "stdcall";
   case CC_X86FastCall:
     return "fastcall";
   case CC_X86_64SysV:
     return "sysv_abi";
   case CC_Win64:
     return "ms_abi";
   case CC_Swift:
     return "swiftcall";
   case CC_SwiftAsync:
     return "swiftasynccall";
   }
   llvm_unreachable("bad calling convention");
 }
 
 void CXXNameMangler::mangleExtFunctionInfo(const FunctionType *T) {
   // Fast path.
   if (T->getExtInfo() == FunctionType::ExtInfo())
     return;
 
   // Vendor-specific qualifiers are emitted in reverse alphabetical order.
   // This will get more complicated in the future if we mangle other
   // things here; but for now, since we mangle ns_returns_retained as
   // a qualifier on the result type, we can get away with this:
   StringRef CCQualifier = getCallingConvQualifierName(T->getExtInfo().getCC());
   if (!CCQualifier.empty())
     mangleVendorQualifier(CCQualifier);
 
   // FIXME: regparm
   // FIXME: noreturn
 }
 
 void
 CXXNameMangler::mangleExtParameterInfo(FunctionProtoType::ExtParameterInfo PI) {
   // Vendor-specific qualifiers are emitted in reverse alphabetical order.
 
   // Note that these are *not* substitution candidates.  Demanglers might
   // have trouble with this if the parameter type is fully substituted.
 
   switch (PI.getABI()) {
   case ParameterABI::Ordinary:
     break;
 
   // All of these start with "swift", so they come before "ns_consumed".
   case ParameterABI::SwiftContext:
   case ParameterABI::SwiftAsyncContext:
   case ParameterABI::SwiftErrorResult:
   case ParameterABI::SwiftIndirectResult:
     mangleVendorQualifier(getParameterABISpelling(PI.getABI()));
     break;
   }
 
   if (PI.isConsumed())
     mangleVendorQualifier("ns_consumed");
 
   if (PI.isNoEscape())
     mangleVendorQualifier("noescape");
 }
 
 // <type>          ::= <function-type>
 // <function-type> ::= [<CV-qualifiers>] F [Y]
 //                      <bare-function-type> [<ref-qualifier>] E
 void CXXNameMangler::mangleType(const FunctionProtoType *T) {
   mangleExtFunctionInfo(T);
 
   // Mangle CV-qualifiers, if present.  These are 'this' qualifiers,
   // e.g. "const" in "int (A::*)() const".
   mangleQualifiers(T->getMethodQuals());
 
   // Mangle instantiation-dependent exception-specification, if present,
   // per cxx-abi-dev proposal on 2016-10-11.
   if (T->hasInstantiationDependentExceptionSpec()) {
     if (isComputedNoexcept(T->getExceptionSpecType())) {
       Out << "DO";
       mangleExpression(T->getNoexceptExpr());
       Out << "E";
     } else {
       assert(T->getExceptionSpecType() == EST_Dynamic);
       Out << "Dw";
       for (auto ExceptTy : T->exceptions())
         mangleType(ExceptTy);
       Out << "E";
     }
   } else if (T->isNothrow()) {
     Out << "Do";
   }
 
   Out << 'F';
 
   // FIXME: We don't have enough information in the AST to produce the 'Y'
   // encoding for extern "C" function types.
   mangleBareFunctionType(T, /*MangleReturnType=*/true);
 
   // Mangle the ref-qualifier, if present.
   mangleRefQualifier(T->getRefQualifier());
 
   Out << 'E';
 }
 
 void CXXNameMangler::mangleType(const FunctionNoProtoType *T) {
   // Function types without prototypes can arise when mangling a function type
   // within an overloadable function in C. We mangle these as the absence of any
   // parameter types (not even an empty parameter list).
   Out << 'F';
 
   FunctionTypeDepthState saved = FunctionTypeDepth.push();
 
   FunctionTypeDepth.enterResultType();
   mangleType(T->getReturnType());
   FunctionTypeDepth.leaveResultType();
 
   FunctionTypeDepth.pop(saved);
   Out << 'E';
 }
 
 void CXXNameMangler::mangleBareFunctionType(const FunctionProtoType *Proto,
                                             bool MangleReturnType,
                                             const FunctionDecl *FD) {
   // Record that we're in a function type.  See mangleFunctionParam
   // for details on what we're trying to achieve here.
   FunctionTypeDepthState saved = FunctionTypeDepth.push();
 
   // <bare-function-type> ::= <signature type>+
   if (MangleReturnType) {
     FunctionTypeDepth.enterResultType();
 
     // Mangle ns_returns_retained as an order-sensitive qualifier here.
     if (Proto->getExtInfo().getProducesResult() && FD == nullptr)
       mangleVendorQualifier("ns_returns_retained");
 
     // Mangle the return type without any direct ARC ownership qualifiers.
     QualType ReturnTy = Proto->getReturnType();
     if (ReturnTy.getObjCLifetime()) {
       auto SplitReturnTy = ReturnTy.split();
       SplitReturnTy.Quals.removeObjCLifetime();
       ReturnTy = getASTContext().getQualifiedType(SplitReturnTy);
     }
     mangleType(ReturnTy);
 
     FunctionTypeDepth.leaveResultType();
   }
 
   if (Proto->getNumParams() == 0 && !Proto->isVariadic()) {
     //   <builtin-type> ::= v   # void
     Out << 'v';
 
     FunctionTypeDepth.pop(saved);
     return;
   }
 
   assert(!FD || FD->getNumParams() == Proto->getNumParams());
   for (unsigned I = 0, E = Proto->getNumParams(); I != E; ++I) {
     // Mangle extended parameter info as order-sensitive qualifiers here.
     if (Proto->hasExtParameterInfos() && FD == nullptr) {
       mangleExtParameterInfo(Proto->getExtParameterInfo(I));
     }
 
     // Mangle the type.
     QualType ParamTy = Proto->getParamType(I);
     mangleType(Context.getASTContext().getSignatureParameterType(ParamTy));
 
     if (FD) {
       if (auto *Attr = FD->getParamDecl(I)->getAttr<PassObjectSizeAttr>()) {
         // Attr can only take 1 character, so we can hardcode the length below.
         assert(Attr->getType() <= 9 && Attr->getType() >= 0);
         if (Attr->isDynamic())
           Out << "U25pass_dynamic_object_size" << Attr->getType();
         else
           Out << "U17pass_object_size" << Attr->getType();
       }
     }
   }
 
   FunctionTypeDepth.pop(saved);
 
   // <builtin-type>      ::= z  # ellipsis
   if (Proto->isVariadic())
     Out << 'z';
 }
 
 // <type>            ::= <class-enum-type>
 // <class-enum-type> ::= <name>
 void CXXNameMangler::mangleType(const UnresolvedUsingType *T) {
   mangleName(T->getDecl());
 }
 
 // <type>            ::= <class-enum-type>
 // <class-enum-type> ::= <name>
 void CXXNameMangler::mangleType(const EnumType *T) {
   mangleType(static_cast<const TagType*>(T));
 }
 void CXXNameMangler::mangleType(const RecordType *T) {
   mangleType(static_cast<const TagType*>(T));
 }
 void CXXNameMangler::mangleType(const TagType *T) {
   mangleName(T->getDecl());
 }
 
 // <type>       ::= <array-type>
 // <array-type> ::= A <positive dimension number> _ <element type>
 //              ::= A [<dimension expression>] _ <element type>
 void CXXNameMangler::mangleType(const ConstantArrayType *T) {
   Out << 'A' << T->getSize() << '_';
   mangleType(T->getElementType());
 }
 void CXXNameMangler::mangleType(const VariableArrayType *T) {
   Out << 'A';
   // decayed vla types (size 0) will just be skipped.
   if (T->getSizeExpr())
     mangleExpression(T->getSizeExpr());
   Out << '_';
   mangleType(T->getElementType());
 }
 void CXXNameMangler::mangleType(const DependentSizedArrayType *T) {
   Out << 'A';
   // A DependentSizedArrayType might not have size expression as below
   //
   // template<int ...N> int arr[] = {N...};
   if (T->getSizeExpr())
     mangleExpression(T->getSizeExpr());
   Out << '_';
   mangleType(T->getElementType());
 }
 void CXXNameMangler::mangleType(const IncompleteArrayType *T) {
   Out << "A_";
   mangleType(T->getElementType());
 }
 
 // <type>                   ::= <pointer-to-member-type>
 // <pointer-to-member-type> ::= M <class type> <member type>
 void CXXNameMangler::mangleType(const MemberPointerType *T) {
   Out << 'M';
   mangleType(QualType(T->getClass(), 0));
   QualType PointeeType = T->getPointeeType();
   if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(PointeeType)) {
     mangleType(FPT);
 
     // Itanium C++ ABI 5.1.8:
     //
     //   The type of a non-static member function is considered to be different,
     //   for the purposes of substitution, from the type of a namespace-scope or
     //   static member function whose type appears similar. The types of two
     //   non-static member functions are considered to be different, for the
     //   purposes of substitution, if the functions are members of different
     //   classes. In other words, for the purposes of substitution, the class of
     //   which the function is a member is considered part of the type of
     //   function.
 
     // Given that we already substitute member function pointers as a
     // whole, the net effect of this rule is just to unconditionally
     // suppress substitution on the function type in a member pointer.
     // We increment the SeqID here to emulate adding an entry to the
     // substitution table.
     ++SeqID;
   } else
     mangleType(PointeeType);
 }
 
 // <type>           ::= <template-param>
 void CXXNameMangler::mangleType(const TemplateTypeParmType *T) {
   mangleTemplateParameter(T->getDepth(), T->getIndex());
 }
 
 // <type>           ::= <template-param>
 void CXXNameMangler::mangleType(const SubstTemplateTypeParmPackType *T) {
   // FIXME: not clear how to mangle this!
   // template <class T...> class A {
   //   template <class U...> void foo(T(*)(U) x...);
   // };
   Out << "_SUBSTPACK_";
 }
 
 // <type> ::= P <type>   # pointer-to
 void CXXNameMangler::mangleType(const PointerType *T) {
   Out << 'P';
   mangleType(T->getPointeeType());
 }
 void CXXNameMangler::mangleType(const ObjCObjectPointerType *T) {
   Out << 'P';
   mangleType(T->getPointeeType());
 }
 
 // <type> ::= R <type>   # reference-to
 void CXXNameMangler::mangleType(const LValueReferenceType *T) {
   Out << 'R';
   mangleType(T->getPointeeType());
 }
 
 // <type> ::= O <type>   # rvalue reference-to (C++0x)
 void CXXNameMangler::mangleType(const RValueReferenceType *T) {
   Out << 'O';
   mangleType(T->getPointeeType());
 }
 
 // <type> ::= C <type>   # complex pair (C 2000)
 void CXXNameMangler::mangleType(const ComplexType *T) {
   Out << 'C';
   mangleType(T->getElementType());
 }
 
 // ARM's ABI for Neon vector types specifies that they should be mangled as
 // if they are structs (to match ARM's initial implementation).  The
 // vector type must be one of the special types predefined by ARM.
 void CXXNameMangler::mangleNeonVectorType(const VectorType *T) {
   QualType EltType = T->getElementType();
   assert(EltType->isBuiltinType() && "Neon vector element not a BuiltinType");
   const char *EltName = nullptr;
   if (T->getVectorKind() == VectorType::NeonPolyVector) {
     switch (cast<BuiltinType>(EltType)->getKind()) {
     case BuiltinType::SChar:
     case BuiltinType::UChar:
       EltName = "poly8_t";
       break;
     case BuiltinType::Short:
     case BuiltinType::UShort:
       EltName = "poly16_t";
       break;
     case BuiltinType::LongLong:
     case BuiltinType::ULongLong:
       EltName = "poly64_t";
       break;
     default: llvm_unreachable("unexpected Neon polynomial vector element type");
     }
   } else {
     switch (cast<BuiltinType>(EltType)->getKind()) {
     case BuiltinType::SChar:     EltName = "int8_t"; break;
     case BuiltinType::UChar:     EltName = "uint8_t"; break;
     case BuiltinType::Short:     EltName = "int16_t"; break;
     case BuiltinType::UShort:    EltName = "uint16_t"; break;
     case BuiltinType::Int:       EltName = "int32_t"; break;
     case BuiltinType::UInt:      EltName = "uint32_t"; break;
     case BuiltinType::LongLong:  EltName = "int64_t"; break;
     case BuiltinType::ULongLong: EltName = "uint64_t"; break;
     case BuiltinType::Double:    EltName = "float64_t"; break;
     case BuiltinType::Float:     EltName = "float32_t"; break;
     case BuiltinType::Half:      EltName = "float16_t"; break;
     case BuiltinType::BFloat16:  EltName = "bfloat16_t"; break;
     default:
       llvm_unreachable("unexpected Neon vector element type");
     }
   }
   const char *BaseName = nullptr;
   unsigned BitSize = (T->getNumElements() *
                       getASTContext().getTypeSize(EltType));
   if (BitSize == 64)
     BaseName = "__simd64_";
   else {
     assert(BitSize == 128 && "Neon vector type not 64 or 128 bits");
     BaseName = "__simd128_";
   }
   Out << strlen(BaseName) + strlen(EltName);
   Out << BaseName << EltName;
 }
 
 void CXXNameMangler::mangleNeonVectorType(const DependentVectorType *T) {
   DiagnosticsEngine &Diags = Context.getDiags();
   unsigned DiagID = Diags.getCustomDiagID(
       DiagnosticsEngine::Error,
       "cannot mangle this dependent neon vector type yet");
   Diags.Report(T->getAttributeLoc(), DiagID);
 }
 
 static StringRef mangleAArch64VectorBase(const BuiltinType *EltType) {
   switch (EltType->getKind()) {
   case BuiltinType::SChar:
     return "Int8";
   case BuiltinType::Short:
     return "Int16";
   case BuiltinType::Int:
     return "Int32";
   case BuiltinType::Long:
   case BuiltinType::LongLong:
     return "Int64";
   case BuiltinType::UChar:
     return "Uint8";
   case BuiltinType::UShort:
     return "Uint16";
   case BuiltinType::UInt:
     return "Uint32";
   case BuiltinType::ULong:
   case BuiltinType::ULongLong:
     return "Uint64";
   case BuiltinType::Half:
     return "Float16";
   case BuiltinType::Float:
     return "Float32";
   case BuiltinType::Double:
     return "Float64";
   case BuiltinType::BFloat16:
     return "Bfloat16";
   default:
     llvm_unreachable("Unexpected vector element base type");
   }
 }
 
 // AArch64's ABI for Neon vector types specifies that they should be mangled as
 // the equivalent internal name. The vector type must be one of the special
 // types predefined by ARM.
 void CXXNameMangler::mangleAArch64NeonVectorType(const VectorType *T) {
   QualType EltType = T->getElementType();
   assert(EltType->isBuiltinType() && "Neon vector element not a BuiltinType");
   unsigned BitSize =
       (T->getNumElements() * getASTContext().getTypeSize(EltType));
   (void)BitSize; // Silence warning.
 
   assert((BitSize == 64 || BitSize == 128) &&
          "Neon vector type not 64 or 128 bits");
 
   StringRef EltName;
   if (T->getVectorKind() == VectorType::NeonPolyVector) {
     switch (cast<BuiltinType>(EltType)->getKind()) {
     case BuiltinType::UChar:
       EltName = "Poly8";
       break;
     case BuiltinType::UShort:
       EltName = "Poly16";
       break;
     case BuiltinType::ULong:
     case BuiltinType::ULongLong:
       EltName = "Poly64";
       break;
     default:
       llvm_unreachable("unexpected Neon polynomial vector element type");
     }
   } else
     EltName = mangleAArch64VectorBase(cast<BuiltinType>(EltType));
 
   std::string TypeName =
       ("__" + EltName + "x" + Twine(T->getNumElements()) + "_t").str();
   Out << TypeName.length() << TypeName;
 }
 void CXXNameMangler::mangleAArch64NeonVectorType(const DependentVectorType *T) {
   DiagnosticsEngine &Diags = Context.getDiags();
   unsigned DiagID = Diags.getCustomDiagID(
       DiagnosticsEngine::Error,
       "cannot mangle this dependent neon vector type yet");
   Diags.Report(T->getAttributeLoc(), DiagID);
 }
 
 // The AArch64 ACLE specifies that fixed-length SVE vector and predicate types
 // defined with the 'arm_sve_vector_bits' attribute map to the same AAPCS64
 // type as the sizeless variants.
 //
 // The mangling scheme for VLS types is implemented as a "pseudo" template:
 //
 //   '__SVE_VLS<<type>, <vector length>>'
 //
 // Combining the existing SVE type and a specific vector length (in bits).
 // For example:
 //
 //   typedef __SVInt32_t foo __attribute__((arm_sve_vector_bits(512)));
 //
 // is described as '__SVE_VLS<__SVInt32_t, 512u>' and mangled as:
 //
 //   "9__SVE_VLSI" + base type mangling + "Lj" + __ARM_FEATURE_SVE_BITS + "EE"
 //
 //   i.e. 9__SVE_VLSIu11__SVInt32_tLj512EE
 //
 // The latest ACLE specification (00bet5) does not contain details of this
 // mangling scheme, it will be specified in the next revision. The mangling
 // scheme is otherwise defined in the appendices to the Procedure Call Standard
 // for the Arm Architecture, see
 // https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#appendix-c-mangling
 void CXXNameMangler::mangleAArch64FixedSveVectorType(const VectorType *T) {
   assert((T->getVectorKind() == VectorType::SveFixedLengthDataVector ||
           T->getVectorKind() == VectorType::SveFixedLengthPredicateVector) &&
          "expected fixed-length SVE vector!");
 
   QualType EltType = T->getElementType();
   assert(EltType->isBuiltinType() &&
          "expected builtin type for fixed-length SVE vector!");
 
   StringRef TypeName;
   switch (cast<BuiltinType>(EltType)->getKind()) {
   case BuiltinType::SChar:
     TypeName = "__SVInt8_t";
     break;
   case BuiltinType::UChar: {
     if (T->getVectorKind() == VectorType::SveFixedLengthDataVector)
       TypeName = "__SVUint8_t";
     else
       TypeName = "__SVBool_t";
     break;
   }
   case BuiltinType::Short:
     TypeName = "__SVInt16_t";
     break;
   case BuiltinType::UShort:
     TypeName = "__SVUint16_t";
     break;
   case BuiltinType::Int:
     TypeName = "__SVInt32_t";
     break;
   case BuiltinType::UInt:
     TypeName = "__SVUint32_t";
     break;
   case BuiltinType::Long:
     TypeName = "__SVInt64_t";
     break;
   case BuiltinType::ULong:
     TypeName = "__SVUint64_t";
     break;
   case BuiltinType::Half:
     TypeName = "__SVFloat16_t";
     break;
   case BuiltinType::Float:
     TypeName = "__SVFloat32_t";
     break;
   case BuiltinType::Double:
     TypeName = "__SVFloat64_t";
     break;
   case BuiltinType::BFloat16:
     TypeName = "__SVBfloat16_t";
     break;
   default:
     llvm_unreachable("unexpected element type for fixed-length SVE vector!");
   }
 
   unsigned VecSizeInBits = getASTContext().getTypeInfo(T).Width;
 
   if (T->getVectorKind() == VectorType::SveFixedLengthPredicateVector)
     VecSizeInBits *= 8;
 
   Out << "9__SVE_VLSI" << 'u' << TypeName.size() << TypeName << "Lj"
       << VecSizeInBits << "EE";
 }
 
 void CXXNameMangler::mangleAArch64FixedSveVectorType(
     const DependentVectorType *T) {
   DiagnosticsEngine &Diags = Context.getDiags();
   unsigned DiagID = Diags.getCustomDiagID(
       DiagnosticsEngine::Error,
       "cannot mangle this dependent fixed-length SVE vector type yet");
   Diags.Report(T->getAttributeLoc(), DiagID);
 }
 
 // GNU extension: vector types
 // <type>                  ::= <vector-type>
 // <vector-type>           ::= Dv <positive dimension number> _
 //                                    <extended element type>
 //                         ::= Dv [<dimension expression>] _ <element type>
 // <extended element type> ::= <element type>
 //                         ::= p # AltiVec vector pixel
 //                         ::= b # Altivec vector bool
 void CXXNameMangler::mangleType(const VectorType *T) {
   if ((T->getVectorKind() == VectorType::NeonVector ||
        T->getVectorKind() == VectorType::NeonPolyVector)) {
     llvm::Triple Target = getASTContext().getTargetInfo().getTriple();
     llvm::Triple::ArchType Arch =
         getASTContext().getTargetInfo().getTriple().getArch();
     if ((Arch == llvm::Triple::aarch64 ||
          Arch == llvm::Triple::aarch64_be) && !Target.isOSDarwin())
       mangleAArch64NeonVectorType(T);
     else
       mangleNeonVectorType(T);
     return;
   } else if (T->getVectorKind() == VectorType::SveFixedLengthDataVector ||
              T->getVectorKind() == VectorType::SveFixedLengthPredicateVector) {
     mangleAArch64FixedSveVectorType(T);
     return;
   }
   Out << "Dv" << T->getNumElements() << '_';
   if (T->getVectorKind() == VectorType::AltiVecPixel)
     Out << 'p';
   else if (T->getVectorKind() == VectorType::AltiVecBool)
     Out << 'b';
   else
     mangleType(T->getElementType());
 }
 
 void CXXNameMangler::mangleType(const DependentVectorType *T) {
   if ((T->getVectorKind() == VectorType::NeonVector ||
        T->getVectorKind() == VectorType::NeonPolyVector)) {
     llvm::Triple Target = getASTContext().getTargetInfo().getTriple();
     llvm::Triple::ArchType Arch =
         getASTContext().getTargetInfo().getTriple().getArch();
     if ((Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be) &&
         !Target.isOSDarwin())
       mangleAArch64NeonVectorType(T);
     else
       mangleNeonVectorType(T);
     return;
   } else if (T->getVectorKind() == VectorType::SveFixedLengthDataVector ||
              T->getVectorKind() == VectorType::SveFixedLengthPredicateVector) {
     mangleAArch64FixedSveVectorType(T);
     return;
   }
 
   Out << "Dv";
   mangleExpression(T->getSizeExpr());
   Out << '_';
   if (T->getVectorKind() == VectorType::AltiVecPixel)
     Out << 'p';
   else if (T->getVectorKind() == VectorType::AltiVecBool)
     Out << 'b';
   else
     mangleType(T->getElementType());
 }
 
 void CXXNameMangler::mangleType(const ExtVectorType *T) {
   mangleType(static_cast<const VectorType*>(T));
 }
 void CXXNameMangler::mangleType(const DependentSizedExtVectorType *T) {
   Out << "Dv";
   mangleExpression(T->getSizeExpr());
   Out << '_';
   mangleType(T->getElementType());
 }
 
 void CXXNameMangler::mangleType(const ConstantMatrixType *T) {
   // Mangle matrix types as a vendor extended type:
   // u<Len>matrix_typeI<Rows><Columns><element type>E
 
   StringRef VendorQualifier = "matrix_type";
   Out << "u" << VendorQualifier.size() << VendorQualifier;
 
   Out << "I";
   auto &ASTCtx = getASTContext();
   unsigned BitWidth = ASTCtx.getTypeSize(ASTCtx.getSizeType());
   llvm::APSInt Rows(BitWidth);
   Rows = T->getNumRows();
   mangleIntegerLiteral(ASTCtx.getSizeType(), Rows);
   llvm::APSInt Columns(BitWidth);
   Columns = T->getNumColumns();
   mangleIntegerLiteral(ASTCtx.getSizeType(), Columns);
   mangleType(T->getElementType());
   Out << "E";
 }
 
 void CXXNameMangler::mangleType(const DependentSizedMatrixType *T) {
   // Mangle matrix types as a vendor extended type:
   // u<Len>matrix_typeI<row expr><column expr><element type>E
   StringRef VendorQualifier = "matrix_type";
   Out << "u" << VendorQualifier.size() << VendorQualifier;
 
   Out << "I";
   mangleTemplateArgExpr(T->getRowExpr());
   mangleTemplateArgExpr(T->getColumnExpr());
   mangleType(T->getElementType());
   Out << "E";
 }
 
 void CXXNameMangler::mangleType(const DependentAddressSpaceType *T) {
   SplitQualType split = T->getPointeeType().split();
   mangleQualifiers(split.Quals, T);
   mangleType(QualType(split.Ty, 0));
 }
 
 void CXXNameMangler::mangleType(const PackExpansionType *T) {
   // <type>  ::= Dp <type>          # pack expansion (C++0x)
   Out << "Dp";
   mangleType(T->getPattern());
 }
 
 void CXXNameMangler::mangleType(const ObjCInterfaceType *T) {
   mangleSourceName(T->getDecl()->getIdentifier());
 }
 
 void CXXNameMangler::mangleType(const ObjCObjectType *T) {
   // Treat __kindof as a vendor extended type qualifier.
   if (T->isKindOfType())
     Out << "U8__kindof";
 
   if (!T->qual_empty()) {
     // Mangle protocol qualifiers.
     SmallString<64> QualStr;
     llvm::raw_svector_ostream QualOS(QualStr);
     QualOS << "objcproto";
     for (const auto *I : T->quals()) {
       StringRef name = I->getName();
       QualOS << name.size() << name;
     }
     Out << 'U' << QualStr.size() << QualStr;
   }
 
   mangleType(T->getBaseType());
 
   if (T->isSpecialized()) {
     // Mangle type arguments as I <type>+ E
     Out << 'I';
     for (auto typeArg : T->getTypeArgs())
       mangleType(typeArg);
     Out << 'E';
   }
 }
 
 void CXXNameMangler::mangleType(const BlockPointerType *T) {
   Out << "U13block_pointer";
   mangleType(T->getPointeeType());
 }
 
 void CXXNameMangler::mangleType(const InjectedClassNameType *T) {
   // Mangle injected class name types as if the user had written the
   // specialization out fully.  It may not actually be possible to see
   // this mangling, though.
   mangleType(T->getInjectedSpecializationType());
 }
 
 void CXXNameMangler::mangleType(const TemplateSpecializationType *T) {
   if (TemplateDecl *TD = T->getTemplateName().getAsTemplateDecl()) {
     mangleTemplateName(TD, T->getArgs(), T->getNumArgs());
   } else {
     if (mangleSubstitution(QualType(T, 0)))
       return;
 
     mangleTemplatePrefix(T->getTemplateName());
 
     // FIXME: GCC does not appear to mangle the template arguments when
     // the template in question is a dependent template name. Should we
     // emulate that badness?
     mangleTemplateArgs(T->getTemplateName(), T->getArgs(), T->getNumArgs());
     addSubstitution(QualType(T, 0));
   }
 }
 
 void CXXNameMangler::mangleType(const DependentNameType *T) {
   // Proposal by cxx-abi-dev, 2014-03-26
   // <class-enum-type> ::= <name>    # non-dependent or dependent type name or
   //                                 # dependent elaborated type specifier using
   //                                 # 'typename'
   //                   ::= Ts <name> # dependent elaborated type specifier using
   //                                 # 'struct' or 'class'
   //                   ::= Tu <name> # dependent elaborated type specifier using
   //                                 # 'union'
   //                   ::= Te <name> # dependent elaborated type specifier using
   //                                 # 'enum'
   switch (T->getKeyword()) {
     case ETK_None:
     case ETK_Typename:
       break;
     case ETK_Struct:
     case ETK_Class:
     case ETK_Interface:
       Out << "Ts";
       break;
     case ETK_Union:
       Out << "Tu";
       break;
     case ETK_Enum:
       Out << "Te";
       break;
   }
   // Typename types are always nested
   Out << 'N';
   manglePrefix(T->getQualifier());
   mangleSourceName(T->getIdentifier());
   Out << 'E';
 }
 
 void CXXNameMangler::mangleType(const DependentTemplateSpecializationType *T) {
   // Dependently-scoped template types are nested if they have a prefix.
   Out << 'N';
 
   // TODO: avoid making this TemplateName.
   TemplateName Prefix =
     getASTContext().getDependentTemplateName(T->getQualifier(),
                                              T->getIdentifier());
   mangleTemplatePrefix(Prefix);
 
   // FIXME: GCC does not appear to mangle the template arguments when
   // the template in question is a dependent template name. Should we
   // emulate that badness?
   mangleTemplateArgs(Prefix, T->getArgs(), T->getNumArgs());
   Out << 'E';
 }
 
 void CXXNameMangler::mangleType(const TypeOfType *T) {
   // FIXME: this is pretty unsatisfactory, but there isn't an obvious
   // "extension with parameters" mangling.
   Out << "u6typeof";
 }
 
 void CXXNameMangler::mangleType(const TypeOfExprType *T) {
   // FIXME: this is pretty unsatisfactory, but there isn't an obvious
   // "extension with parameters" mangling.
   Out << "u6typeof";
 }
 
 void CXXNameMangler::mangleType(const DecltypeType *T) {
   Expr *E = T->getUnderlyingExpr();
 
   // type ::= Dt <expression> E  # decltype of an id-expression
   //                             #   or class member access
   //      ::= DT <expression> E  # decltype of an expression
 
   // This purports to be an exhaustive list of id-expressions and
   // class member accesses.  Note that we do not ignore parentheses;
   // parentheses change the semantics of decltype for these
   // expressions (and cause the mangler to use the other form).
   if (isa<DeclRefExpr>(E) ||
       isa<MemberExpr>(E) ||
       isa<UnresolvedLookupExpr>(E) ||
       isa<DependentScopeDeclRefExpr>(E) ||
       isa<CXXDependentScopeMemberExpr>(E) ||
       isa<UnresolvedMemberExpr>(E))
     Out << "Dt";
   else
     Out << "DT";
   mangleExpression(E);
   Out << 'E';
 }
 
 void CXXNameMangler::mangleType(const UnaryTransformType *T) {
   // If this is dependent, we need to record that. If not, we simply
   // mangle it as the underlying type since they are equivalent.
   if (T->isDependentType()) {
     Out << 'U';
 
     switch (T->getUTTKind()) {
       case UnaryTransformType::EnumUnderlyingType:
         Out << "3eut";
         break;
     }
   }
 
   mangleType(T->getBaseType());
 }
 
 void CXXNameMangler::mangleType(const AutoType *T) {
   assert(T->getDeducedType().isNull() &&
          "Deduced AutoType shouldn't be handled here!");
   assert(T->getKeyword() != AutoTypeKeyword::GNUAutoType &&
          "shouldn't need to mangle __auto_type!");
   // <builtin-type> ::= Da # auto
   //                ::= Dc # decltype(auto)
   Out << (T->isDecltypeAuto() ? "Dc" : "Da");
 }
 
 void CXXNameMangler::mangleType(const DeducedTemplateSpecializationType *T) {
   QualType Deduced = T->getDeducedType();
   if (!Deduced.isNull())
     return mangleType(Deduced);
 
   TemplateDecl *TD = T->getTemplateName().getAsTemplateDecl();
   assert(TD && "shouldn't form deduced TST unless we know we have a template");
 
   if (mangleSubstitution(TD))
     return;
 
   mangleName(GlobalDecl(TD));
   addSubstitution(TD);
 }
 
 void CXXNameMangler::mangleType(const AtomicType *T) {
   // <type> ::= U <source-name> <type>  # vendor extended type qualifier
   // (Until there's a standardized mangling...)
   Out << "U7_Atomic";
   mangleType(T->getValueType());
 }
 
 void CXXNameMangler::mangleType(const PipeType *T) {
   // Pipe type mangling rules are described in SPIR 2.0 specification
   // A.1 Data types and A.3 Summary of changes
   // <type> ::= 8ocl_pipe
   Out << "8ocl_pipe";
 }
 
 void CXXNameMangler::mangleType(const BitIntType *T) {
   // 5.1.5.2 Builtin types
   // <type> ::= DB <number | instantiation-dependent expression> _
   //        ::= DU <number | instantiation-dependent expression> _
   Out << "D" << (T->isUnsigned() ? "U" : "B") << T->getNumBits() << "_";
 }
 
 void CXXNameMangler::mangleType(const DependentBitIntType *T) {
   // 5.1.5.2 Builtin types
   // <type> ::= DB <number | instantiation-dependent expression> _
   //        ::= DU <number | instantiation-dependent expression> _
   Out << "D" << (T->isUnsigned() ? "U" : "B");
   mangleExpression(T->getNumBitsExpr());
   Out << "_";
 }
 
 void CXXNameMangler::mangleIntegerLiteral(QualType T,
                                           const llvm::APSInt &Value) {
   //  <expr-primary> ::= L <type> <value number> E # integer literal
   Out << 'L';
 
   mangleType(T);
   if (T->isBooleanType()) {
     // Boolean values are encoded as 0/1.
     Out << (Value.getBoolValue() ? '1' : '0');
   } else {
     mangleNumber(Value);
   }
   Out << 'E';
 
 }
 
 void CXXNameMangler::mangleMemberExprBase(const Expr *Base, bool IsArrow) {
   // Ignore member expressions involving anonymous unions.
   while (const auto *RT = Base->getType()->getAs<RecordType>()) {
     if (!RT->getDecl()->isAnonymousStructOrUnion())
       break;
     const auto *ME = dyn_cast<MemberExpr>(Base);
     if (!ME)
       break;
     Base = ME->getBase();
     IsArrow = ME->isArrow();
   }
 
   if (Base->isImplicitCXXThis()) {
     // Note: GCC mangles member expressions to the implicit 'this' as
     // *this., whereas we represent them as this->. The Itanium C++ ABI
     // does not specify anything here, so we follow GCC.
     Out << "dtdefpT";
   } else {
     Out << (IsArrow ? "pt" : "dt");
     mangleExpression(Base);
   }
 }
 
 /// Mangles a member expression.
 void CXXNameMangler::mangleMemberExpr(const Expr *base,
                                       bool isArrow,
                                       NestedNameSpecifier *qualifier,
                                       NamedDecl *firstQualifierLookup,
                                       DeclarationName member,
                                       const TemplateArgumentLoc *TemplateArgs,
                                       unsigned NumTemplateArgs,
                                       unsigned arity) {
   // <expression> ::= dt <expression> <unresolved-name>
   //              ::= pt <expression> <unresolved-name>
   if (base)
     mangleMemberExprBase(base, isArrow);
   mangleUnresolvedName(qualifier, member, TemplateArgs, NumTemplateArgs, arity);
 }
 
 /// Look at the callee of the given call expression and determine if
 /// it's a parenthesized id-expression which would have triggered ADL
 /// otherwise.
 static bool isParenthesizedADLCallee(const CallExpr *call) {
   const Expr *callee = call->getCallee();
   const Expr *fn = callee->IgnoreParens();
 
   // Must be parenthesized.  IgnoreParens() skips __extension__ nodes,
   // too, but for those to appear in the callee, it would have to be
   // parenthesized.
   if (callee == fn) return false;
 
   // Must be an unresolved lookup.
   const UnresolvedLookupExpr *lookup = dyn_cast<UnresolvedLookupExpr>(fn);
   if (!lookup) return false;
 
   assert(!lookup->requiresADL());
 
   // Must be an unqualified lookup.
   if (lookup->getQualifier()) return false;
 
   // Must not have found a class member.  Note that if one is a class
   // member, they're all class members.
   if (lookup->getNumDecls() > 0 &&
       (*lookup->decls_begin())->isCXXClassMember())
     return false;
 
   // Otherwise, ADL would have been triggered.
   return true;
 }
 
 void CXXNameMangler::mangleCastExpression(const Expr *E, StringRef CastEncoding) {
   const ExplicitCastExpr *ECE = cast<ExplicitCastExpr>(E);
   Out << CastEncoding;
   mangleType(ECE->getType());
   mangleExpression(ECE->getSubExpr());
 }
 
 void CXXNameMangler::mangleInitListElements(const InitListExpr *InitList) {
   if (auto *Syntactic = InitList->getSyntacticForm())
     InitList = Syntactic;
   for (unsigned i = 0, e = InitList->getNumInits(); i != e; ++i)
     mangleExpression(InitList->getInit(i));
 }
 
 void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity,
                                       bool AsTemplateArg) {
   // <expression> ::= <unary operator-name> <expression>
   //              ::= <binary operator-name> <expression> <expression>
   //              ::= <trinary operator-name> <expression> <expression> <expression>
   //              ::= cv <type> expression           # conversion with one argument
   //              ::= cv <type> _ <expression>* E # conversion with a different number of arguments
   //              ::= dc <type> <expression>         # dynamic_cast<type> (expression)
   //              ::= sc <type> <expression>         # static_cast<type> (expression)
   //              ::= cc <type> <expression>         # const_cast<type> (expression)
   //              ::= rc <type> <expression>         # reinterpret_cast<type> (expression)
   //              ::= st <type>                      # sizeof (a type)
   //              ::= at <type>                      # alignof (a type)
   //              ::= <template-param>
   //              ::= <function-param>
   //              ::= fpT                            # 'this' expression (part of <function-param>)
   //              ::= sr <type> <unqualified-name>                   # dependent name
   //              ::= sr <type> <unqualified-name> <template-args>   # dependent template-id
   //              ::= ds <expression> <expression>                   # expr.*expr
   //              ::= sZ <template-param>                            # size of a parameter pack
   //              ::= sZ <function-param>    # size of a function parameter pack
   //              ::= u <source-name> <template-arg>* E # vendor extended expression
   //              ::= <expr-primary>
   // <expr-primary> ::= L <type> <value number> E    # integer literal
   //                ::= L <type> <value float> E     # floating literal
   //                ::= L <type> <string type> E     # string literal
   //                ::= L <nullptr type> E           # nullptr literal "LDnE"
   //                ::= L <pointer type> 0 E         # null pointer template argument
   //                ::= L <type> <real-part float> _ <imag-part float> E    # complex floating point literal (C99); not used by clang
   //                ::= L <mangled-name> E           # external name
   QualType ImplicitlyConvertedToType;
 
   // A top-level expression that's not <expr-primary> needs to be wrapped in
   // X...E in a template arg.
   bool IsPrimaryExpr = true;
   auto NotPrimaryExpr = [&] {
     if (AsTemplateArg && IsPrimaryExpr)
       Out << 'X';
     IsPrimaryExpr = false;
   };
 
   auto MangleDeclRefExpr = [&](const NamedDecl *D) {
     switch (D->getKind()) {
     default:
       //  <expr-primary> ::= L <mangled-name> E # external name
       Out << 'L';
       mangle(D);
       Out << 'E';
       break;
 
     case Decl::ParmVar:
       NotPrimaryExpr();
       mangleFunctionParam(cast<ParmVarDecl>(D));
       break;
 
     case Decl::EnumConstant: {
       // <expr-primary>
       const EnumConstantDecl *ED = cast<EnumConstantDecl>(D);
       mangleIntegerLiteral(ED->getType(), ED->getInitVal());
       break;
     }
 
     case Decl::NonTypeTemplateParm:
       NotPrimaryExpr();
       const NonTypeTemplateParmDecl *PD = cast<NonTypeTemplateParmDecl>(D);
       mangleTemplateParameter(PD->getDepth(), PD->getIndex());
       break;
     }
   };
 
   // 'goto recurse' is used when handling a simple "unwrapping" node which
   // produces no output, where ImplicitlyConvertedToType and AsTemplateArg need
   // to be preserved.
 recurse:
   switch (E->getStmtClass()) {
   case Expr::NoStmtClass:
 #define ABSTRACT_STMT(Type)
 #define EXPR(Type, Base)
 #define STMT(Type, Base) \
   case Expr::Type##Class:
 #include "clang/AST/StmtNodes.inc"
     // fallthrough
 
   // These all can only appear in local or variable-initialization
   // contexts and so should never appear in a mangling.
   case Expr::AddrLabelExprClass:
   case Expr::DesignatedInitUpdateExprClass:
   case Expr::ImplicitValueInitExprClass:
   case Expr::ArrayInitLoopExprClass:
   case Expr::ArrayInitIndexExprClass:
   case Expr::NoInitExprClass:
   case Expr::ParenListExprClass:
   case Expr::MSPropertyRefExprClass:
   case Expr::MSPropertySubscriptExprClass:
   case Expr::TypoExprClass: // This should no longer exist in the AST by now.
   case Expr::RecoveryExprClass:
   case Expr::OMPArraySectionExprClass:
   case Expr::OMPArrayShapingExprClass:
   case Expr::OMPIteratorExprClass:
   case Expr::CXXInheritedCtorInitExprClass:
     llvm_unreachable("unexpected statement kind");
 
   case Expr::ConstantExprClass:
     E = cast<ConstantExpr>(E)->getSubExpr();
     goto recurse;
 
   // FIXME: invent manglings for all these.
   case Expr::BlockExprClass:
   case Expr::ChooseExprClass:
   case Expr::CompoundLiteralExprClass:
   case Expr::ExtVectorElementExprClass:
   case Expr::GenericSelectionExprClass:
   case Expr::ObjCEncodeExprClass:
   case Expr::ObjCIsaExprClass:
   case Expr::ObjCIvarRefExprClass:
   case Expr::ObjCMessageExprClass:
   case Expr::ObjCPropertyRefExprClass:
   case Expr::ObjCProtocolExprClass:
   case Expr::ObjCSelectorExprClass:
   case Expr::ObjCStringLiteralClass:
   case Expr::ObjCBoxedExprClass:
   case Expr::ObjCArrayLiteralClass:
   case Expr::ObjCDictionaryLiteralClass:
   case Expr::ObjCSubscriptRefExprClass:
   case Expr::ObjCIndirectCopyRestoreExprClass:
   case Expr::ObjCAvailabilityCheckExprClass:
   case Expr::OffsetOfExprClass:
   case Expr::PredefinedExprClass:
   case Expr::ShuffleVectorExprClass:
   case Expr::ConvertVectorExprClass:
   case Expr::StmtExprClass:
   case Expr::TypeTraitExprClass:
   case Expr::RequiresExprClass:
   case Expr::ArrayTypeTraitExprClass:
   case Expr::ExpressionTraitExprClass:
   case Expr::VAArgExprClass:
   case Expr::CUDAKernelCallExprClass:
   case Expr::AsTypeExprClass:
   case Expr::PseudoObjectExprClass:
   case Expr::AtomicExprClass:
   case Expr::SourceLocExprClass:
   case Expr::BuiltinBitCastExprClass:
   {
     NotPrimaryExpr();
     if (!NullOut) {
       // As bad as this diagnostic is, it's better than crashing.
       DiagnosticsEngine &Diags = Context.getDiags();
       unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
                                        "cannot yet mangle expression type %0");
       Diags.Report(E->getExprLoc(), DiagID)
         << E->getStmtClassName() << E->getSourceRange();
       return;
     }
     break;
   }
 
   case Expr::CXXUuidofExprClass: {
     NotPrimaryExpr();
     const CXXUuidofExpr *UE = cast<CXXUuidofExpr>(E);
     // As of clang 12, uuidof uses the vendor extended expression
     // mangling. Previously, it used a special-cased nonstandard extension.
     if (Context.getASTContext().getLangOpts().getClangABICompat() >
         LangOptions::ClangABI::Ver11) {
       Out << "u8__uuidof";
       if (UE->isTypeOperand())
         mangleType(UE->getTypeOperand(Context.getASTContext()));
       else
         mangleTemplateArgExpr(UE->getExprOperand());
       Out << 'E';
     } else {
       if (UE->isTypeOperand()) {
         QualType UuidT = UE->getTypeOperand(Context.getASTContext());
         Out << "u8__uuidoft";
         mangleType(UuidT);
       } else {
         Expr *UuidExp = UE->getExprOperand();
         Out << "u8__uuidofz";
         mangleExpression(UuidExp);
       }
     }
     break;
   }
 
   // Even gcc-4.5 doesn't mangle this.
   case Expr::BinaryConditionalOperatorClass: {
     NotPrimaryExpr();
     DiagnosticsEngine &Diags = Context.getDiags();
     unsigned DiagID =
       Diags.getCustomDiagID(DiagnosticsEngine::Error,
                 "?: operator with omitted middle operand cannot be mangled");
     Diags.Report(E->getExprLoc(), DiagID)
       << E->getStmtClassName() << E->getSourceRange();
     return;
   }
 
   // These are used for internal purposes and cannot be meaningfully mangled.
   case Expr::OpaqueValueExprClass:
     llvm_unreachable("cannot mangle opaque value; mangling wrong thing?");
 
   case Expr::InitListExprClass: {
     NotPrimaryExpr();
     Out << "il";
     mangleInitListElements(cast<InitListExpr>(E));
     Out << "E";
     break;
   }
 
   case Expr::DesignatedInitExprClass: {
     NotPrimaryExpr();
     auto *DIE = cast<DesignatedInitExpr>(E);
     for (const auto &Designator : DIE->designators()) {
       if (Designator.isFieldDesignator()) {
         Out << "di";
         mangleSourceName(Designator.getFieldName());
       } else if (Designator.isArrayDesignator()) {
         Out << "dx";
         mangleExpression(DIE->getArrayIndex(Designator));
       } else {
         assert(Designator.isArrayRangeDesignator() &&
                "unknown designator kind");
         Out << "dX";
         mangleExpression(DIE->getArrayRangeStart(Designator));
         mangleExpression(DIE->getArrayRangeEnd(Designator));
       }
     }
     mangleExpression(DIE->getInit());
     break;
   }
 
   case Expr::CXXDefaultArgExprClass:
     E = cast<CXXDefaultArgExpr>(E)->getExpr();
     goto recurse;
 
   case Expr::CXXDefaultInitExprClass:
     E = cast<CXXDefaultInitExpr>(E)->getExpr();
     goto recurse;
 
   case Expr::CXXStdInitializerListExprClass:
     E = cast<CXXStdInitializerListExpr>(E)->getSubExpr();
     goto recurse;
 
   case Expr::SubstNonTypeTemplateParmExprClass:
     E = cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement();
     goto recurse;
 
   case Expr::UserDefinedLiteralClass:
     // We follow g++'s approach of mangling a UDL as a call to the literal
     // operator.
   case Expr::CXXMemberCallExprClass: // fallthrough
   case Expr::CallExprClass: {
     NotPrimaryExpr();
     const CallExpr *CE = cast<CallExpr>(E);
 
     // <expression> ::= cp <simple-id> <expression>* E
     // We use this mangling only when the call would use ADL except
     // for being parenthesized.  Per discussion with David
     // Vandervoorde, 2011.04.25.
     if (isParenthesizedADLCallee(CE)) {
       Out << "cp";
       // The callee here is a parenthesized UnresolvedLookupExpr with
       // no qualifier and should always get mangled as a <simple-id>
       // anyway.
 
     // <expression> ::= cl <expression>* E
     } else {
       Out << "cl";
     }
 
     unsigned CallArity = CE->getNumArgs();
     for (const Expr *Arg : CE->arguments())
       if (isa<PackExpansionExpr>(Arg))
         CallArity = UnknownArity;
 
     mangleExpression(CE->getCallee(), CallArity);
     for (const Expr *Arg : CE->arguments())
       mangleExpression(Arg);
     Out << 'E';
     break;
   }
 
   case Expr::CXXNewExprClass: {
     NotPrimaryExpr();
     const CXXNewExpr *New = cast<CXXNewExpr>(E);
     if (New->isGlobalNew()) Out << "gs";
     Out << (New->isArray() ? "na" : "nw");
     for (CXXNewExpr::const_arg_iterator I = New->placement_arg_begin(),
            E = New->placement_arg_end(); I != E; ++I)
       mangleExpression(*I);
     Out << '_';
     mangleType(New->getAllocatedType());
     if (New->hasInitializer()) {
       if (New->getInitializationStyle() == CXXNewExpr::ListInit)
         Out << "il";
       else
         Out << "pi";
       const Expr *Init = New->getInitializer();
       if (const CXXConstructExpr *CCE = dyn_cast<CXXConstructExpr>(Init)) {
         // Directly inline the initializers.
         for (CXXConstructExpr::const_arg_iterator I = CCE->arg_begin(),
                                                   E = CCE->arg_end();
              I != E; ++I)
           mangleExpression(*I);
       } else if (const ParenListExpr *PLE = dyn_cast<ParenListExpr>(Init)) {
         for (unsigned i = 0, e = PLE->getNumExprs(); i != e; ++i)
           mangleExpression(PLE->getExpr(i));
       } else if (New->getInitializationStyle() == CXXNewExpr::ListInit &&
                  isa<InitListExpr>(Init)) {
         // Only take InitListExprs apart for list-initialization.
         mangleInitListElements(cast<InitListExpr>(Init));
       } else
         mangleExpression(Init);
     }
     Out << 'E';
     break;
   }
 
   case Expr::CXXPseudoDestructorExprClass: {
     NotPrimaryExpr();
     const auto *PDE = cast<CXXPseudoDestructorExpr>(E);
     if (const Expr *Base = PDE->getBase())
       mangleMemberExprBase(Base, PDE->isArrow());
     NestedNameSpecifier *Qualifier = PDE->getQualifier();
     if (TypeSourceInfo *ScopeInfo = PDE->getScopeTypeInfo()) {
       if (Qualifier) {
         mangleUnresolvedPrefix(Qualifier,
                                /*recursive=*/true);
         mangleUnresolvedTypeOrSimpleId(ScopeInfo->getType());
         Out << 'E';
       } else {
         Out << "sr";
         if (!mangleUnresolvedTypeOrSimpleId(ScopeInfo->getType()))
           Out << 'E';
       }
     } else if (Qualifier) {
       mangleUnresolvedPrefix(Qualifier);
     }
     // <base-unresolved-name> ::= dn <destructor-name>
     Out << "dn";
     QualType DestroyedType = PDE->getDestroyedType();
     mangleUnresolvedTypeOrSimpleId(DestroyedType);
     break;
   }
 
   case Expr::MemberExprClass: {
     NotPrimaryExpr();
     const MemberExpr *ME = cast<MemberExpr>(E);
     mangleMemberExpr(ME->getBase(), ME->isArrow(),
                      ME->getQualifier(), nullptr,
                      ME->getMemberDecl()->getDeclName(),
                      ME->getTemplateArgs(), ME->getNumTemplateArgs(),
                      Arity);
     break;
   }
 
   case Expr::UnresolvedMemberExprClass: {
     NotPrimaryExpr();
     const UnresolvedMemberExpr *ME = cast<UnresolvedMemberExpr>(E);
     mangleMemberExpr(ME->isImplicitAccess() ? nullptr : ME->getBase(),
                      ME->isArrow(), ME->getQualifier(), nullptr,
                      ME->getMemberName(),
                      ME->getTemplateArgs(), ME->getNumTemplateArgs(),
                      Arity);
     break;
   }
 
   case Expr::CXXDependentScopeMemberExprClass: {
     NotPrimaryExpr();
     const CXXDependentScopeMemberExpr *ME
       = cast<CXXDependentScopeMemberExpr>(E);
     mangleMemberExpr(ME->isImplicitAccess() ? nullptr : ME->getBase(),
                      ME->isArrow(), ME->getQualifier(),
                      ME->getFirstQualifierFoundInScope(),
                      ME->getMember(),
                      ME->getTemplateArgs(), ME->getNumTemplateArgs(),
                      Arity);
     break;
   }
 
   case Expr::UnresolvedLookupExprClass: {
     NotPrimaryExpr();
     const UnresolvedLookupExpr *ULE = cast<UnresolvedLookupExpr>(E);
     mangleUnresolvedName(ULE->getQualifier(), ULE->getName(),
                          ULE->getTemplateArgs(), ULE->getNumTemplateArgs(),
                          Arity);
     break;
   }
 
   case Expr::CXXUnresolvedConstructExprClass: {
     NotPrimaryExpr();
     const CXXUnresolvedConstructExpr *CE = cast<CXXUnresolvedConstructExpr>(E);
     unsigned N = CE->getNumArgs();
 
     if (CE->isListInitialization()) {
       assert(N == 1 && "unexpected form for list initialization");
       auto *IL = cast<InitListExpr>(CE->getArg(0));
       Out << "tl";
       mangleType(CE->getType());
       mangleInitListElements(IL);
       Out << "E";
       break;
     }
 
     Out << "cv";
     mangleType(CE->getType());
     if (N != 1) Out << '_';
     for (unsigned I = 0; I != N; ++I) mangleExpression(CE->getArg(I));
     if (N != 1) Out << 'E';
     break;
   }
 
   case Expr::CXXConstructExprClass: {
     // An implicit cast is silent, thus may contain <expr-primary>.
     const auto *CE = cast<CXXConstructExpr>(E);
     if (!CE->isListInitialization() || CE->isStdInitListInitialization()) {
       assert(
           CE->getNumArgs() >= 1 &&
           (CE->getNumArgs() == 1 || isa<CXXDefaultArgExpr>(CE->getArg(1))) &&
           "implicit CXXConstructExpr must have one argument");
       E = cast<CXXConstructExpr>(E)->getArg(0);
       goto recurse;
     }
     NotPrimaryExpr();
     Out << "il";
     for (auto *E : CE->arguments())
       mangleExpression(E);
     Out << "E";
     break;
   }
 
   case Expr::CXXTemporaryObjectExprClass: {
     NotPrimaryExpr();
     const auto *CE = cast<CXXTemporaryObjectExpr>(E);
     unsigned N = CE->getNumArgs();
     bool List = CE->isListInitialization();
 
     if (List)
       Out << "tl";
     else
       Out << "cv";
     mangleType(CE->getType());
     if (!List && N != 1)
       Out << '_';
     if (CE->isStdInitListInitialization()) {
       // We implicitly created a std::initializer_list<T> for the first argument
       // of a constructor of type U in an expression of the form U{a, b, c}.
       // Strip all the semantic gunk off the initializer list.
       auto *SILE =
           cast<CXXStdInitializerListExpr>(CE->getArg(0)->IgnoreImplicit());
       auto *ILE = cast<InitListExpr>(SILE->getSubExpr()->IgnoreImplicit());
       mangleInitListElements(ILE);
     } else {
       for (auto *E : CE->arguments())
         mangleExpression(E);
     }
     if (List || N != 1)
       Out << 'E';
     break;
   }
 
   case Expr::CXXScalarValueInitExprClass:
     NotPrimaryExpr();
     Out << "cv";
     mangleType(E->getType());
     Out << "_E";
     break;
 
   case Expr::CXXNoexceptExprClass:
     NotPrimaryExpr();
     Out << "nx";
     mangleExpression(cast<CXXNoexceptExpr>(E)->getOperand());
     break;
 
   case Expr::UnaryExprOrTypeTraitExprClass: {
     // Non-instantiation-dependent traits are an <expr-primary> integer literal.
     const UnaryExprOrTypeTraitExpr *SAE = cast<UnaryExprOrTypeTraitExpr>(E);
 
     if (!SAE->isInstantiationDependent()) {
       // Itanium C++ ABI:
       //   If the operand of a sizeof or alignof operator is not
       //   instantiation-dependent it is encoded as an integer literal
       //   reflecting the result of the operator.
       //
       //   If the result of the operator is implicitly converted to a known
       //   integer type, that type is used for the literal; otherwise, the type
       //   of std::size_t or std::ptrdiff_t is used.
       QualType T = (ImplicitlyConvertedToType.isNull() ||
                     !ImplicitlyConvertedToType->isIntegerType())? SAE->getType()
                                                     : ImplicitlyConvertedToType;
       llvm::APSInt V = SAE->EvaluateKnownConstInt(Context.getASTContext());
       mangleIntegerLiteral(T, V);
       break;
     }
 
     NotPrimaryExpr(); // But otherwise, they are not.
 
     auto MangleAlignofSizeofArg = [&] {
       if (SAE->isArgumentType()) {
         Out << 't';
         mangleType(SAE->getArgumentType());
       } else {
         Out << 'z';
         mangleExpression(SAE->getArgumentExpr());
       }
     };
 
     switch(SAE->getKind()) {
     case UETT_SizeOf:
       Out << 's';
       MangleAlignofSizeofArg();
       break;
     case UETT_PreferredAlignOf:
       // As of clang 12, we mangle __alignof__ differently than alignof. (They
       // have acted differently since Clang 8, but were previously mangled the
       // same.)
       if (Context.getASTContext().getLangOpts().getClangABICompat() >
           LangOptions::ClangABI::Ver11) {
         Out << "u11__alignof__";
         if (SAE->isArgumentType())
           mangleType(SAE->getArgumentType());
         else
           mangleTemplateArgExpr(SAE->getArgumentExpr());
         Out << 'E';
         break;
       }
       LLVM_FALLTHROUGH;
     case UETT_AlignOf:
       Out << 'a';
       MangleAlignofSizeofArg();
       break;
     case UETT_VecStep: {
       DiagnosticsEngine &Diags = Context.getDiags();
       unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
                                      "cannot yet mangle vec_step expression");
       Diags.Report(DiagID);
       return;
     }
     case UETT_OpenMPRequiredSimdAlign: {
       DiagnosticsEngine &Diags = Context.getDiags();
       unsigned DiagID = Diags.getCustomDiagID(
           DiagnosticsEngine::Error,
           "cannot yet mangle __builtin_omp_required_simd_align expression");
       Diags.Report(DiagID);
       return;
     }
     }
     break;
   }
 
   case Expr::CXXThrowExprClass: {
     NotPrimaryExpr();
     const CXXThrowExpr *TE = cast<CXXThrowExpr>(E);
     //  <expression> ::= tw <expression>  # throw expression
     //               ::= tr               # rethrow
     if (TE->getSubExpr()) {
       Out << "tw";
       mangleExpression(TE->getSubExpr());
     } else {
       Out << "tr";
     }
     break;
   }
 
   case Expr::CXXTypeidExprClass: {
     NotPrimaryExpr();
     const CXXTypeidExpr *TIE = cast<CXXTypeidExpr>(E);
     //  <expression> ::= ti <type>        # typeid (type)
     //               ::= te <expression>  # typeid (expression)
     if (TIE->isTypeOperand()) {
       Out << "ti";
       mangleType(TIE->getTypeOperand(Context.getASTContext()));
     } else {
       Out << "te";
       mangleExpression(TIE->getExprOperand());
     }
     break;
   }
 
   case Expr::CXXDeleteExprClass: {
     NotPrimaryExpr();
     const CXXDeleteExpr *DE = cast<CXXDeleteExpr>(E);
     //  <expression> ::= [gs] dl <expression>  # [::] delete expr
     //               ::= [gs] da <expression>  # [::] delete [] expr
     if (DE->isGlobalDelete()) Out << "gs";
     Out << (DE->isArrayForm() ? "da" : "dl");
     mangleExpression(DE->getArgument());
     break;
   }
 
   case Expr::UnaryOperatorClass: {
     NotPrimaryExpr();
     const UnaryOperator *UO = cast<UnaryOperator>(E);
     mangleOperatorName(UnaryOperator::getOverloadedOperator(UO->getOpcode()),
                        /*Arity=*/1);
     mangleExpression(UO->getSubExpr());
     break;
   }
 
   case Expr::ArraySubscriptExprClass: {
     NotPrimaryExpr();
     const ArraySubscriptExpr *AE = cast<ArraySubscriptExpr>(E);
 
     // Array subscript is treated as a syntactically weird form of
     // binary operator.
     Out << "ix";
     mangleExpression(AE->getLHS());
     mangleExpression(AE->getRHS());
     break;
   }
 
   case Expr::MatrixSubscriptExprClass: {
     NotPrimaryExpr();
     const MatrixSubscriptExpr *ME = cast<MatrixSubscriptExpr>(E);
     Out << "ixix";
     mangleExpression(ME->getBase());
     mangleExpression(ME->getRowIdx());
     mangleExpression(ME->getColumnIdx());
     break;
   }
 
   case Expr::CompoundAssignOperatorClass: // fallthrough
   case Expr::BinaryOperatorClass: {
     NotPrimaryExpr();
     const BinaryOperator *BO = cast<BinaryOperator>(E);
     if (BO->getOpcode() == BO_PtrMemD)
       Out << "ds";
     else
       mangleOperatorName(BinaryOperator::getOverloadedOperator(BO->getOpcode()),
                          /*Arity=*/2);
     mangleExpression(BO->getLHS());
     mangleExpression(BO->getRHS());
     break;
   }
 
   case Expr::CXXRewrittenBinaryOperatorClass: {
     NotPrimaryExpr();
     // The mangled form represents the original syntax.
     CXXRewrittenBinaryOperator::DecomposedForm Decomposed =
         cast<CXXRewrittenBinaryOperator>(E)->getDecomposedForm();
     mangleOperatorName(BinaryOperator::getOverloadedOperator(Decomposed.Opcode),
                        /*Arity=*/2);
     mangleExpression(Decomposed.LHS);
     mangleExpression(Decomposed.RHS);
     break;
   }
 
   case Expr::ConditionalOperatorClass: {
     NotPrimaryExpr();
     const ConditionalOperator *CO = cast<ConditionalOperator>(E);
     mangleOperatorName(OO_Conditional, /*Arity=*/3);
     mangleExpression(CO->getCond());
     mangleExpression(CO->getLHS(), Arity);
     mangleExpression(CO->getRHS(), Arity);
     break;
   }
 
   case Expr::ImplicitCastExprClass: {
     ImplicitlyConvertedToType = E->getType();
     E = cast<ImplicitCastExpr>(E)->getSubExpr();
     goto recurse;
   }
 
   case Expr::ObjCBridgedCastExprClass: {
     NotPrimaryExpr();
     // Mangle ownership casts as a vendor extended operator __bridge,
     // __bridge_transfer, or __bridge_retain.
     StringRef Kind = cast<ObjCBridgedCastExpr>(E)->getBridgeKindName();
     Out << "v1U" << Kind.size() << Kind;
     mangleCastExpression(E, "cv");
     break;
   }
 
   case Expr::CStyleCastExprClass:
     NotPrimaryExpr();
     mangleCastExpression(E, "cv");
     break;
 
   case Expr::CXXFunctionalCastExprClass: {
     NotPrimaryExpr();
     auto *Sub = cast<ExplicitCastExpr>(E)->getSubExpr()->IgnoreImplicit();
     // FIXME: Add isImplicit to CXXConstructExpr.
     if (auto *CCE = dyn_cast<CXXConstructExpr>(Sub))
       if (CCE->getParenOrBraceRange().isInvalid())
         Sub = CCE->getArg(0)->IgnoreImplicit();
     if (auto *StdInitList = dyn_cast<CXXStdInitializerListExpr>(Sub))
       Sub = StdInitList->getSubExpr()->IgnoreImplicit();
     if (auto *IL = dyn_cast<InitListExpr>(Sub)) {
       Out << "tl";
       mangleType(E->getType());
       mangleInitListElements(IL);
       Out << "E";
     } else {
       mangleCastExpression(E, "cv");
     }
     break;
   }
 
   case Expr::CXXStaticCastExprClass:
     NotPrimaryExpr();
     mangleCastExpression(E, "sc");
     break;
   case Expr::CXXDynamicCastExprClass:
     NotPrimaryExpr();
     mangleCastExpression(E, "dc");
     break;
   case Expr::CXXReinterpretCastExprClass:
     NotPrimaryExpr();
     mangleCastExpression(E, "rc");
     break;
   case Expr::CXXConstCastExprClass:
     NotPrimaryExpr();
     mangleCastExpression(E, "cc");
     break;
   case Expr::CXXAddrspaceCastExprClass:
     NotPrimaryExpr();
     mangleCastExpression(E, "ac");
     break;
 
   case Expr::CXXOperatorCallExprClass: {
     NotPrimaryExpr();
     const CXXOperatorCallExpr *CE = cast<CXXOperatorCallExpr>(E);
     unsigned NumArgs = CE->getNumArgs();
     // A CXXOperatorCallExpr for OO_Arrow models only semantics, not syntax
     // (the enclosing MemberExpr covers the syntactic portion).
     if (CE->getOperator() != OO_Arrow)
       mangleOperatorName(CE->getOperator(), /*Arity=*/NumArgs);
     // Mangle the arguments.
     for (unsigned i = 0; i != NumArgs; ++i)
       mangleExpression(CE->getArg(i));
     break;
   }
 
   case Expr::ParenExprClass:
     E = cast<ParenExpr>(E)->getSubExpr();
     goto recurse;
 
   case Expr::ConceptSpecializationExprClass: {
     //  <expr-primary> ::= L <mangled-name> E # external name
     Out << "L_Z";
     auto *CSE = cast<ConceptSpecializationExpr>(E);
     mangleTemplateName(CSE->getNamedConcept(),
                        CSE->getTemplateArguments().data(),
                        CSE->getTemplateArguments().size());
     Out << 'E';
     break;
   }
 
   case Expr::DeclRefExprClass:
     // MangleDeclRefExpr helper handles primary-vs-nonprimary
     MangleDeclRefExpr(cast<DeclRefExpr>(E)->getDecl());
     break;
 
   case Expr::SubstNonTypeTemplateParmPackExprClass:
     NotPrimaryExpr();
     // FIXME: not clear how to mangle this!
     // template <unsigned N...> class A {
     //   template <class U...> void foo(U (&x)[N]...);
     // };
     Out << "_SUBSTPACK_";
     break;
 
   case Expr::FunctionParmPackExprClass: {
     NotPrimaryExpr();
     // FIXME: not clear how to mangle this!
     const FunctionParmPackExpr *FPPE = cast<FunctionParmPackExpr>(E);
     Out << "v110_SUBSTPACK";
     MangleDeclRefExpr(FPPE->getParameterPack());
     break;
   }
 
   case Expr::DependentScopeDeclRefExprClass: {
     NotPrimaryExpr();
     const DependentScopeDeclRefExpr *DRE = cast<DependentScopeDeclRefExpr>(E);
     mangleUnresolvedName(DRE->getQualifier(), DRE->getDeclName(),
                          DRE->getTemplateArgs(), DRE->getNumTemplateArgs(),
                          Arity);
     break;
   }
 
   case Expr::CXXBindTemporaryExprClass:
     E = cast<CXXBindTemporaryExpr>(E)->getSubExpr();
     goto recurse;
 
   case Expr::ExprWithCleanupsClass:
     E = cast<ExprWithCleanups>(E)->getSubExpr();
     goto recurse;
 
   case Expr::FloatingLiteralClass: {
     // <expr-primary>
     const FloatingLiteral *FL = cast<FloatingLiteral>(E);
     mangleFloatLiteral(FL->getType(), FL->getValue());
     break;
   }
 
   case Expr::FixedPointLiteralClass:
     // Currently unimplemented -- might be <expr-primary> in future?
     mangleFixedPointLiteral();
     break;
 
   case Expr::CharacterLiteralClass:
     // <expr-primary>
     Out << 'L';
     mangleType(E->getType());
     Out << cast<CharacterLiteral>(E)->getValue();
     Out << 'E';
     break;
 
   // FIXME. __objc_yes/__objc_no are mangled same as true/false
   case Expr::ObjCBoolLiteralExprClass:
     // <expr-primary>
     Out << "Lb";
     Out << (cast<ObjCBoolLiteralExpr>(E)->getValue() ? '1' : '0');
     Out << 'E';
     break;
 
   case Expr::CXXBoolLiteralExprClass:
     // <expr-primary>
     Out << "Lb";
     Out << (cast<CXXBoolLiteralExpr>(E)->getValue() ? '1' : '0');
     Out << 'E';
     break;
 
   case Expr::IntegerLiteralClass: {
     // <expr-primary>
     llvm::APSInt Value(cast<IntegerLiteral>(E)->getValue());
     if (E->getType()->isSignedIntegerType())
       Value.setIsSigned(true);
     mangleIntegerLiteral(E->getType(), Value);
     break;
   }
 
   case Expr::ImaginaryLiteralClass: {
     // <expr-primary>
     const ImaginaryLiteral *IE = cast<ImaginaryLiteral>(E);
     // Mangle as if a complex literal.
     // Proposal from David Vandevoorde, 2010.06.30.
     Out << 'L';
     mangleType(E->getType());
     if (const FloatingLiteral *Imag =
           dyn_cast<FloatingLiteral>(IE->getSubExpr())) {
       // Mangle a floating-point zero of the appropriate type.
       mangleFloat(llvm::APFloat(Imag->getValue().getSemantics()));
       Out << '_';
       mangleFloat(Imag->getValue());
     } else {
       Out << "0_";
       llvm::APSInt Value(cast<IntegerLiteral>(IE->getSubExpr())->getValue());
       if (IE->getSubExpr()->getType()->isSignedIntegerType())
         Value.setIsSigned(true);
       mangleNumber(Value);
     }
     Out << 'E';
     break;
   }
 
   case Expr::StringLiteralClass: {
     // <expr-primary>
     // Revised proposal from David Vandervoorde, 2010.07.15.
     Out << 'L';
     assert(isa<ConstantArrayType>(E->getType()));
     mangleType(E->getType());
     Out << 'E';
     break;
   }
 
   case Expr::GNUNullExprClass:
     // <expr-primary>
     // Mangle as if an integer literal 0.
     mangleIntegerLiteral(E->getType(), llvm::APSInt(32));
     break;
 
   case Expr::CXXNullPtrLiteralExprClass: {
     // <expr-primary>
     Out << "LDnE";
     break;
   }
 
   case Expr::LambdaExprClass: {
     // A lambda-expression can't appear in the signature of an
     // externally-visible declaration, so there's no standard mangling for
     // this, but mangling as a literal of the closure type seems reasonable.
     Out << "L";
     mangleType(Context.getASTContext().getRecordType(cast<LambdaExpr>(E)->getLambdaClass()));
     Out << "E";
     break;
   }
 
   case Expr::PackExpansionExprClass:
     NotPrimaryExpr();
     Out << "sp";
     mangleExpression(cast<PackExpansionExpr>(E)->getPattern());
     break;
 
   case Expr::SizeOfPackExprClass: {
     NotPrimaryExpr();
     auto *SPE = cast<SizeOfPackExpr>(E);
     if (SPE->isPartiallySubstituted()) {
       Out << "sP";
       for (const auto &A : SPE->getPartialArguments())
         mangleTemplateArg(A, false);
       Out << "E";
       break;
     }
 
     Out << "sZ";
     const NamedDecl *Pack = SPE->getPack();
     if (const TemplateTypeParmDecl *TTP = dyn_cast<TemplateTypeParmDecl>(Pack))
       mangleTemplateParameter(TTP->getDepth(), TTP->getIndex());
     else if (const NonTypeTemplateParmDecl *NTTP
                 = dyn_cast<NonTypeTemplateParmDecl>(Pack))
       mangleTemplateParameter(NTTP->getDepth(), NTTP->getIndex());
     else if (const TemplateTemplateParmDecl *TempTP
                                     = dyn_cast<TemplateTemplateParmDecl>(Pack))
       mangleTemplateParameter(TempTP->getDepth(), TempTP->getIndex());
     else
       mangleFunctionParam(cast<ParmVarDecl>(Pack));
     break;
   }
 
   case Expr::MaterializeTemporaryExprClass:
     E = cast<MaterializeTemporaryExpr>(E)->getSubExpr();
     goto recurse;
 
   case Expr::CXXFoldExprClass: {
     NotPrimaryExpr();
     auto *FE = cast<CXXFoldExpr>(E);
     if (FE->isLeftFold())
       Out << (FE->getInit() ? "fL" : "fl");
     else
       Out << (FE->getInit() ? "fR" : "fr");
 
     if (FE->getOperator() == BO_PtrMemD)
       Out << "ds";
     else
       mangleOperatorName(
           BinaryOperator::getOverloadedOperator(FE->getOperator()),
           /*Arity=*/2);
 
     if (FE->getLHS())
       mangleExpression(FE->getLHS());
     if (FE->getRHS())
       mangleExpression(FE->getRHS());
     break;
   }
 
   case Expr::CXXThisExprClass:
     NotPrimaryExpr();
     Out << "fpT";
     break;
 
   case Expr::CoawaitExprClass:
     // FIXME: Propose a non-vendor mangling.
     NotPrimaryExpr();
     Out << "v18co_await";
     mangleExpression(cast<CoawaitExpr>(E)->getOperand());
     break;
 
   case Expr::DependentCoawaitExprClass:
     // FIXME: Propose a non-vendor mangling.
     NotPrimaryExpr();
     Out << "v18co_await";
     mangleExpression(cast<DependentCoawaitExpr>(E)->getOperand());
     break;
 
   case Expr::CoyieldExprClass:
     // FIXME: Propose a non-vendor mangling.
     NotPrimaryExpr();
     Out << "v18co_yield";
     mangleExpression(cast<CoawaitExpr>(E)->getOperand());
     break;
   case Expr::SYCLUniqueStableNameExprClass: {
     const auto *USN = cast<SYCLUniqueStableNameExpr>(E);
     NotPrimaryExpr();
 
     Out << "u33__builtin_sycl_unique_stable_name";
     mangleType(USN->getTypeSourceInfo()->getType());
 
     Out << "E";
     break;
   }
   }
 
   if (AsTemplateArg && !IsPrimaryExpr)
     Out << 'E';
 }
 
 /// Mangle an expression which refers to a parameter variable.
 ///
 /// <expression>     ::= <function-param>
 /// <function-param> ::= fp <top-level CV-qualifiers> _      # L == 0, I == 0
 /// <function-param> ::= fp <top-level CV-qualifiers>
 ///                      <parameter-2 non-negative number> _ # L == 0, I > 0
 /// <function-param> ::= fL <L-1 non-negative number>
 ///                      p <top-level CV-qualifiers> _       # L > 0, I == 0
 /// <function-param> ::= fL <L-1 non-negative number>
 ///                      p <top-level CV-qualifiers>
 ///                      <I-1 non-negative number> _         # L > 0, I > 0
 ///
 /// L is the nesting depth of the parameter, defined as 1 if the
 /// parameter comes from the innermost function prototype scope
 /// enclosing the current context, 2 if from the next enclosing
 /// function prototype scope, and so on, with one special case: if
 /// we've processed the full parameter clause for the innermost
 /// function type, then L is one less.  This definition conveniently
 /// makes it irrelevant whether a function's result type was written
 /// trailing or leading, but is otherwise overly complicated; the
 /// numbering was first designed without considering references to
 /// parameter in locations other than return types, and then the
 /// mangling had to be generalized without changing the existing
 /// manglings.
 ///
 /// I is the zero-based index of the parameter within its parameter
 /// declaration clause.  Note that the original ABI document describes
 /// this using 1-based ordinals.
 void CXXNameMangler::mangleFunctionParam(const ParmVarDecl *parm) {
   unsigned parmDepth = parm->getFunctionScopeDepth();
   unsigned parmIndex = parm->getFunctionScopeIndex();
 
   // Compute 'L'.
   // parmDepth does not include the declaring function prototype.
   // FunctionTypeDepth does account for that.
   assert(parmDepth < FunctionTypeDepth.getDepth());
   unsigned nestingDepth = FunctionTypeDepth.getDepth() - parmDepth;
   if (FunctionTypeDepth.isInResultType())
     nestingDepth--;
 
   if (nestingDepth == 0) {
     Out << "fp";
   } else {
     Out << "fL" << (nestingDepth - 1) << 'p';
   }
 
   // Top-level qualifiers.  We don't have to worry about arrays here,
   // because parameters declared as arrays should already have been
   // transformed to have pointer type. FIXME: apparently these don't
   // get mangled if used as an rvalue of a known non-class type?
   assert(!parm->getType()->isArrayType()
          && "parameter's type is still an array type?");
 
   if (const DependentAddressSpaceType *DAST =
       dyn_cast<DependentAddressSpaceType>(parm->getType())) {
     mangleQualifiers(DAST->getPointeeType().getQualifiers(), DAST);
   } else {
     mangleQualifiers(parm->getType().getQualifiers());
   }
 
   // Parameter index.
   if (parmIndex != 0) {
     Out << (parmIndex - 1);
   }
   Out << '_';
 }
 
 void CXXNameMangler::mangleCXXCtorType(CXXCtorType T,
                                        const CXXRecordDecl *InheritedFrom) {
   // <ctor-dtor-name> ::= C1  # complete object constructor
   //                  ::= C2  # base object constructor
   //                  ::= CI1 <type> # complete inheriting constructor
   //                  ::= CI2 <type> # base inheriting constructor
   //
   // In addition, C5 is a comdat name with C1 and C2 in it.
   Out << 'C';
   if (InheritedFrom)
     Out << 'I';
   switch (T) {
   case Ctor_Complete:
     Out << '1';
     break;
   case Ctor_Base:
     Out << '2';
     break;
   case Ctor_Comdat:
     Out << '5';
     break;
   case Ctor_DefaultClosure:
   case Ctor_CopyingClosure:
     llvm_unreachable("closure constructors don't exist for the Itanium ABI!");
   }
   if (InheritedFrom)
     mangleName(InheritedFrom);
 }
 
 void CXXNameMangler::mangleCXXDtorType(CXXDtorType T) {
   // <ctor-dtor-name> ::= D0  # deleting destructor
   //                  ::= D1  # complete object destructor
   //                  ::= D2  # base object destructor
   //
   // In addition, D5 is a comdat name with D1, D2 and, if virtual, D0 in it.
   switch (T) {
   case Dtor_Deleting:
     Out << "D0";
     break;
   case Dtor_Complete:
     Out << "D1";
     break;
   case Dtor_Base:
     Out << "D2";
     break;
   case Dtor_Comdat:
     Out << "D5";
     break;
   }
 }
 
 namespace {
 // Helper to provide ancillary information on a template used to mangle its
 // arguments.
 struct TemplateArgManglingInfo {
   TemplateDecl *ResolvedTemplate = nullptr;
   bool SeenPackExpansionIntoNonPack = false;
   const NamedDecl *UnresolvedExpandedPack = nullptr;
 
   TemplateArgManglingInfo(TemplateName TN) {
     if (TemplateDecl *TD = TN.getAsTemplateDecl())
       ResolvedTemplate = TD;
   }
 
   /// Do we need to mangle template arguments with exactly correct types?
   ///
   /// This should be called exactly once for each parameter / argument pair, in
   /// order.
   bool needExactType(unsigned ParamIdx, const TemplateArgument &Arg) {
     // We need correct types when the template-name is unresolved or when it
     // names a template that is able to be overloaded.
     if (!ResolvedTemplate || SeenPackExpansionIntoNonPack)
       return true;
 
     // Move to the next parameter.
     const NamedDecl *Param = UnresolvedExpandedPack;
     if (!Param) {
       assert(ParamIdx < ResolvedTemplate->getTemplateParameters()->size() &&
              "no parameter for argument");
       Param = ResolvedTemplate->getTemplateParameters()->getParam(ParamIdx);
 
       // If we reach an expanded parameter pack whose argument isn't in pack
       // form, that means Sema couldn't figure out which arguments belonged to
       // it, because it contains a pack expansion. Track the expanded pack for
       // all further template arguments until we hit that pack expansion.
       if (Param->isParameterPack() && Arg.getKind() != TemplateArgument::Pack) {
         assert(getExpandedPackSize(Param) &&
                "failed to form pack argument for parameter pack");
         UnresolvedExpandedPack = Param;
       }
     }
 
     // If we encounter a pack argument that is expanded into a non-pack
     // parameter, we can no longer track parameter / argument correspondence,
     // and need to use exact types from this point onwards.
     if (Arg.isPackExpansion() &&
         (!Param->isParameterPack() || UnresolvedExpandedPack)) {
       SeenPackExpansionIntoNonPack = true;
       return true;
     }
 
     // We need exact types for function template arguments because they might be
     // overloaded on template parameter type. As a special case, a member
     // function template of a generic lambda is not overloadable.
     if (auto *FTD = dyn_cast<FunctionTemplateDecl>(ResolvedTemplate)) {
       auto *RD = dyn_cast<CXXRecordDecl>(FTD->getDeclContext());
       if (!RD || !RD->isGenericLambda())
         return true;
     }
 
     // Otherwise, we only need a correct type if the parameter has a deduced
     // type.
     //
     // Note: for an expanded parameter pack, getType() returns the type prior
     // to expansion. We could ask for the expanded type with getExpansionType(),
     // but it doesn't matter because substitution and expansion don't affect
     // whether a deduced type appears in the type.
     auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(Param);
     return NTTP && NTTP->getType()->getContainedDeducedType();
   }
 };
 }
 
 void CXXNameMangler::mangleTemplateArgs(TemplateName TN,
                                         const TemplateArgumentLoc *TemplateArgs,
                                         unsigned NumTemplateArgs) {
   // <template-args> ::= I <template-arg>+ E
   Out << 'I';
   TemplateArgManglingInfo Info(TN);
   for (unsigned i = 0; i != NumTemplateArgs; ++i)
     mangleTemplateArg(TemplateArgs[i].getArgument(),
                       Info.needExactType(i, TemplateArgs[i].getArgument()));
   Out << 'E';
 }
 
 void CXXNameMangler::mangleTemplateArgs(TemplateName TN,
                                         const TemplateArgumentList &AL) {
   // <template-args> ::= I <template-arg>+ E
   Out << 'I';
   TemplateArgManglingInfo Info(TN);
   for (unsigned i = 0, e = AL.size(); i != e; ++i)
     mangleTemplateArg(AL[i], Info.needExactType(i, AL[i]));
   Out << 'E';
 }
 
 void CXXNameMangler::mangleTemplateArgs(TemplateName TN,
                                         const TemplateArgument *TemplateArgs,
                                         unsigned NumTemplateArgs) {
   // <template-args> ::= I <template-arg>+ E
   Out << 'I';
   TemplateArgManglingInfo Info(TN);
   for (unsigned i = 0; i != NumTemplateArgs; ++i)
     mangleTemplateArg(TemplateArgs[i], Info.needExactType(i, TemplateArgs[i]));
   Out << 'E';
 }
 
 void CXXNameMangler::mangleTemplateArg(TemplateArgument A, bool NeedExactType) {
   // <template-arg> ::= <type>              # type or template
   //                ::= X <expression> E    # expression
   //                ::= <expr-primary>      # simple expressions
   //                ::= J <template-arg>* E # argument pack
   if (!A.isInstantiationDependent() || A.isDependent())
     A = Context.getASTContext().getCanonicalTemplateArgument(A);
 
   switch (A.getKind()) {
   case TemplateArgument::Null:
     llvm_unreachable("Cannot mangle NULL template argument");
 
   case TemplateArgument::Type:
     mangleType(A.getAsType());
     break;
   case TemplateArgument::Template:
     // This is mangled as <type>.
     mangleType(A.getAsTemplate());
     break;
   case TemplateArgument::TemplateExpansion:
     // <type>  ::= Dp <type>          # pack expansion (C++0x)
     Out << "Dp";
     mangleType(A.getAsTemplateOrTemplatePattern());
     break;
   case TemplateArgument::Expression:
     mangleTemplateArgExpr(A.getAsExpr());
     break;
   case TemplateArgument::Integral:
     mangleIntegerLiteral(A.getIntegralType(), A.getAsIntegral());
     break;
   case TemplateArgument::Declaration: {
     //  <expr-primary> ::= L <mangled-name> E # external name
     ValueDecl *D = A.getAsDecl();
 
     // Template parameter objects are modeled by reproducing a source form
     // produced as if by aggregate initialization.
     if (A.getParamTypeForDecl()->isRecordType()) {
       auto *TPO = cast<TemplateParamObjectDecl>(D);
       mangleValueInTemplateArg(TPO->getType().getUnqualifiedType(),
                                TPO->getValue(), /*TopLevel=*/true,
                                NeedExactType);
       break;
     }
 
     ASTContext &Ctx = Context.getASTContext();
     APValue Value;
     if (D->isCXXInstanceMember())
       // Simple pointer-to-member with no conversion.
       Value = APValue(D, /*IsDerivedMember=*/false, /*Path=*/{});
     else if (D->getType()->isArrayType() &&
              Ctx.hasSimilarType(Ctx.getDecayedType(D->getType()),
                                 A.getParamTypeForDecl()) &&
              Ctx.getLangOpts().getClangABICompat() >
                  LangOptions::ClangABI::Ver11)
       // Build a value corresponding to this implicit array-to-pointer decay.
       Value = APValue(APValue::LValueBase(D), CharUnits::Zero(),
                       {APValue::LValuePathEntry::ArrayIndex(0)},
                       /*OnePastTheEnd=*/false);
     else
       // Regular pointer or reference to a declaration.
       Value = APValue(APValue::LValueBase(D), CharUnits::Zero(),
                       ArrayRef<APValue::LValuePathEntry>(),
                       /*OnePastTheEnd=*/false);
     mangleValueInTemplateArg(A.getParamTypeForDecl(), Value, /*TopLevel=*/true,
                              NeedExactType);
     break;
   }
   case TemplateArgument::NullPtr: {
     mangleNullPointer(A.getNullPtrType());
     break;
   }
   case TemplateArgument::Pack: {
     //  <template-arg> ::= J <template-arg>* E
     Out << 'J';
     for (const auto &P : A.pack_elements())
       mangleTemplateArg(P, NeedExactType);
     Out << 'E';
   }
   }
 }
 
 void CXXNameMangler::mangleTemplateArgExpr(const Expr *E) {
   ASTContext &Ctx = Context.getASTContext();
   if (Ctx.getLangOpts().getClangABICompat() > LangOptions::ClangABI::Ver11) {
     mangleExpression(E, UnknownArity, /*AsTemplateArg=*/true);
     return;
   }
 
   // Prior to Clang 12, we didn't omit the X .. E around <expr-primary>
   // correctly in cases where the template argument was
   // constructed from an expression rather than an already-evaluated
   // literal. In such a case, we would then e.g. emit 'XLi0EE' instead of
   // 'Li0E'.
   //
   // We did special-case DeclRefExpr to attempt to DTRT for that one
   // expression-kind, but while doing so, unfortunately handled ParmVarDecl
   // (subtype of VarDecl) _incorrectly_, and emitted 'L_Z .. E' instead of
   // the proper 'Xfp_E'.
   E = E->IgnoreParenImpCasts();
   if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E)) {
     const ValueDecl *D = DRE->getDecl();
     if (isa<VarDecl>(D) || isa<FunctionDecl>(D)) {
       Out << 'L';
       mangle(D);
       Out << 'E';
       return;
     }
   }
   Out << 'X';
   mangleExpression(E);
   Out << 'E';
 }
 
 /// Determine whether a given value is equivalent to zero-initialization for
 /// the purpose of discarding a trailing portion of a 'tl' mangling.
 ///
 /// Note that this is not in general equivalent to determining whether the
 /// value has an all-zeroes bit pattern.
 static bool isZeroInitialized(QualType T, const APValue &V) {
   // FIXME: mangleValueInTemplateArg has quadratic time complexity in
   // pathological cases due to using this, but it's a little awkward
   // to do this in linear time in general.
   switch (V.getKind()) {
   case APValue::None:
   case APValue::Indeterminate:
   case APValue::AddrLabelDiff:
     return false;
 
   case APValue::Struct: {
     const CXXRecordDecl *RD = T->getAsCXXRecordDecl();
     assert(RD && "unexpected type for record value");
     unsigned I = 0;
     for (const CXXBaseSpecifier &BS : RD->bases()) {
       if (!isZeroInitialized(BS.getType(), V.getStructBase(I)))
         return false;
       ++I;
     }
     I = 0;
     for (const FieldDecl *FD : RD->fields()) {
       if (!FD->isUnnamedBitfield() &&
           !isZeroInitialized(FD->getType(), V.getStructField(I)))
         return false;
       ++I;
     }
     return true;
   }
 
   case APValue::Union: {
     const CXXRecordDecl *RD = T->getAsCXXRecordDecl();
     assert(RD && "unexpected type for union value");
     // Zero-initialization zeroes the first non-unnamed-bitfield field, if any.
     for (const FieldDecl *FD : RD->fields()) {
       if (!FD->isUnnamedBitfield())
         return V.getUnionField() && declaresSameEntity(FD, V.getUnionField()) &&
                isZeroInitialized(FD->getType(), V.getUnionValue());
     }
     // If there are no fields (other than unnamed bitfields), the value is
     // necessarily zero-initialized.
     return true;
   }
 
   case APValue::Array: {
     QualType ElemT(T->getArrayElementTypeNoTypeQual(), 0);
     for (unsigned I = 0, N = V.getArrayInitializedElts(); I != N; ++I)
       if (!isZeroInitialized(ElemT, V.getArrayInitializedElt(I)))
         return false;
     return !V.hasArrayFiller() || isZeroInitialized(ElemT, V.getArrayFiller());
   }
 
   case APValue::Vector: {
     const VectorType *VT = T->castAs<VectorType>();
     for (unsigned I = 0, N = V.getVectorLength(); I != N; ++I)
       if (!isZeroInitialized(VT->getElementType(), V.getVectorElt(I)))
         return false;
     return true;
   }
 
   case APValue::Int:
     return !V.getInt();
 
   case APValue::Float:
     return V.getFloat().isPosZero();
 
   case APValue::FixedPoint:
     return !V.getFixedPoint().getValue();
 
   case APValue::ComplexFloat:
     return V.getComplexFloatReal().isPosZero() &&
            V.getComplexFloatImag().isPosZero();
 
   case APValue::ComplexInt:
     return !V.getComplexIntReal() && !V.getComplexIntImag();
 
   case APValue::LValue:
     return V.isNullPointer();
 
   case APValue::MemberPointer:
     return !V.getMemberPointerDecl();
   }
 
   llvm_unreachable("Unhandled APValue::ValueKind enum");
 }
 
 static QualType getLValueType(ASTContext &Ctx, const APValue &LV) {
   QualType T = LV.getLValueBase().getType();
   for (APValue::LValuePathEntry E : LV.getLValuePath()) {
     if (const ArrayType *AT = Ctx.getAsArrayType(T))
       T = AT->getElementType();
     else if (const FieldDecl *FD =
                  dyn_cast<FieldDecl>(E.getAsBaseOrMember().getPointer()))
       T = FD->getType();
     else
       T = Ctx.getRecordType(
           cast<CXXRecordDecl>(E.getAsBaseOrMember().getPointer()));
   }
   return T;
 }
 
 void CXXNameMangler::mangleValueInTemplateArg(QualType T, const APValue &V,
                                               bool TopLevel,
                                               bool NeedExactType) {
   // Ignore all top-level cv-qualifiers, to match GCC.
   Qualifiers Quals;
   T = getASTContext().getUnqualifiedArrayType(T, Quals);
 
   // A top-level expression that's not a primary expression is wrapped in X...E.
   bool IsPrimaryExpr = true;
   auto NotPrimaryExpr = [&] {
     if (TopLevel && IsPrimaryExpr)
       Out << 'X';
     IsPrimaryExpr = false;
   };
 
   // Proposed in https://github.com/itanium-cxx-abi/cxx-abi/issues/63.
   switch (V.getKind()) {
   case APValue::None:
   case APValue::Indeterminate:
     Out << 'L';
     mangleType(T);
     Out << 'E';
     break;
 
   case APValue::AddrLabelDiff:
     llvm_unreachable("unexpected value kind in template argument");
 
   case APValue::Struct: {
     const CXXRecordDecl *RD = T->getAsCXXRecordDecl();
     assert(RD && "unexpected type for record value");
 
     // Drop trailing zero-initialized elements.
     llvm::SmallVector<const FieldDecl *, 16> Fields(RD->field_begin(),
                                                     RD->field_end());
     while (
         !Fields.empty() &&
         (Fields.back()->isUnnamedBitfield() ||
          isZeroInitialized(Fields.back()->getType(),
                            V.getStructField(Fields.back()->getFieldIndex())))) {
       Fields.pop_back();
     }
     llvm::ArrayRef<CXXBaseSpecifier> Bases(RD->bases_begin(), RD->bases_end());
     if (Fields.empty()) {
       while (!Bases.empty() &&
              isZeroInitialized(Bases.back().getType(),
                                V.getStructBase(Bases.size() - 1)))
         Bases = Bases.drop_back();
     }
 
     // <expression> ::= tl <type> <braced-expression>* E
     NotPrimaryExpr();
     Out << "tl";
     mangleType(T);
     for (unsigned I = 0, N = Bases.size(); I != N; ++I)
       mangleValueInTemplateArg(Bases[I].getType(), V.getStructBase(I), false);
     for (unsigned I = 0, N = Fields.size(); I != N; ++I) {
       if (Fields[I]->isUnnamedBitfield())
         continue;
       mangleValueInTemplateArg(Fields[I]->getType(),
                                V.getStructField(Fields[I]->getFieldIndex()),
                                false);
     }
     Out << 'E';
     break;
   }
 
   case APValue::Union: {
     assert(T->getAsCXXRecordDecl() && "unexpected type for union value");
     const FieldDecl *FD = V.getUnionField();
 
     if (!FD) {
       Out << 'L';
       mangleType(T);
       Out << 'E';
       break;
     }
 
     // <braced-expression> ::= di <field source-name> <braced-expression>
     NotPrimaryExpr();
     Out << "tl";
     mangleType(T);
     if (!isZeroInitialized(T, V)) {
       Out << "di";
       mangleSourceName(FD->getIdentifier());
       mangleValueInTemplateArg(FD->getType(), V.getUnionValue(), false);
     }
     Out << 'E';
     break;
   }
 
   case APValue::Array: {
     QualType ElemT(T->getArrayElementTypeNoTypeQual(), 0);
 
     NotPrimaryExpr();
     Out << "tl";
     mangleType(T);
 
     // Drop trailing zero-initialized elements.
     unsigned N = V.getArraySize();
     if (!V.hasArrayFiller() || isZeroInitialized(ElemT, V.getArrayFiller())) {
       N = V.getArrayInitializedElts();
       while (N && isZeroInitialized(ElemT, V.getArrayInitializedElt(N - 1)))
         --N;
     }
 
     for (unsigned I = 0; I != N; ++I) {
       const APValue &Elem = I < V.getArrayInitializedElts()
                                 ? V.getArrayInitializedElt(I)
                                 : V.getArrayFiller();
       mangleValueInTemplateArg(ElemT, Elem, false);
     }
     Out << 'E';
     break;
   }
 
   case APValue::Vector: {
     const VectorType *VT = T->castAs<VectorType>();
 
     NotPrimaryExpr();
     Out << "tl";
     mangleType(T);
     unsigned N = V.getVectorLength();
     while (N && isZeroInitialized(VT->getElementType(), V.getVectorElt(N - 1)))
       --N;
     for (unsigned I = 0; I != N; ++I)
       mangleValueInTemplateArg(VT->getElementType(), V.getVectorElt(I), false);
     Out << 'E';
     break;
   }
 
   case APValue::Int:
     mangleIntegerLiteral(T, V.getInt());
     break;
 
   case APValue::Float:
     mangleFloatLiteral(T, V.getFloat());
     break;
 
   case APValue::FixedPoint:
     mangleFixedPointLiteral();
     break;
 
   case APValue::ComplexFloat: {
     const ComplexType *CT = T->castAs<ComplexType>();
     NotPrimaryExpr();
     Out << "tl";
     mangleType(T);
     if (!V.getComplexFloatReal().isPosZero() ||
         !V.getComplexFloatImag().isPosZero())
       mangleFloatLiteral(CT->getElementType(), V.getComplexFloatReal());
     if (!V.getComplexFloatImag().isPosZero())
       mangleFloatLiteral(CT->getElementType(), V.getComplexFloatImag());
     Out << 'E';
     break;
   }
 
   case APValue::ComplexInt: {
     const ComplexType *CT = T->castAs<ComplexType>();
     NotPrimaryExpr();
     Out << "tl";
     mangleType(T);
     if (V.getComplexIntReal().getBoolValue() ||
         V.getComplexIntImag().getBoolValue())
       mangleIntegerLiteral(CT->getElementType(), V.getComplexIntReal());
     if (V.getComplexIntImag().getBoolValue())
       mangleIntegerLiteral(CT->getElementType(), V.getComplexIntImag());
     Out << 'E';
     break;
   }
 
   case APValue::LValue: {
     // Proposed in https://github.com/itanium-cxx-abi/cxx-abi/issues/47.
     assert((T->isPointerType() || T->isReferenceType()) &&
            "unexpected type for LValue template arg");
 
     if (V.isNullPointer()) {
       mangleNullPointer(T);
       break;
     }
 
     APValue::LValueBase B = V.getLValueBase();
     if (!B) {
       // Non-standard mangling for integer cast to a pointer; this can only
       // occur as an extension.
       CharUnits Offset = V.getLValueOffset();
       if (Offset.isZero()) {
         // This is reinterpret_cast<T*>(0), not a null pointer. Mangle this as
         // a cast, because L <type> 0 E means something else.
         NotPrimaryExpr();
         Out << "rc";
         mangleType(T);
         Out << "Li0E";
         if (TopLevel)
           Out << 'E';
       } else {
         Out << "L";
         mangleType(T);
         Out << Offset.getQuantity() << 'E';
       }
       break;
     }
 
     ASTContext &Ctx = Context.getASTContext();
 
     enum { Base, Offset, Path } Kind;
     if (!V.hasLValuePath()) {
       // Mangle as (T*)((char*)&base + N).
       if (T->isReferenceType()) {
         NotPrimaryExpr();
         Out << "decvP";
         mangleType(T->getPointeeType());
       } else {
         NotPrimaryExpr();
         Out << "cv";
         mangleType(T);
       }
       Out << "plcvPcad";
       Kind = Offset;
     } else {
       if (!V.getLValuePath().empty() || V.isLValueOnePastTheEnd()) {
         NotPrimaryExpr();
         // A final conversion to the template parameter's type is usually
         // folded into the 'so' mangling, but we can't do that for 'void*'
         // parameters without introducing collisions.
         if (NeedExactType && T->isVoidPointerType()) {
           Out << "cv";
           mangleType(T);
         }
         if (T->isPointerType())
           Out << "ad";
         Out << "so";
         mangleType(T->isVoidPointerType()
                        ? getLValueType(Ctx, V).getUnqualifiedType()
                        : T->getPointeeType());
         Kind = Path;
       } else {
         if (NeedExactType &&
             !Ctx.hasSameType(T->getPointeeType(), getLValueType(Ctx, V)) &&
             Ctx.getLangOpts().getClangABICompat() >
                 LangOptions::ClangABI::Ver11) {
           NotPrimaryExpr();
           Out << "cv";
           mangleType(T);
         }
         if (T->isPointerType()) {
           NotPrimaryExpr();
           Out << "ad";
         }
         Kind = Base;
       }
     }
 
     QualType TypeSoFar = B.getType();
     if (auto *VD = B.dyn_cast<const ValueDecl*>()) {
       Out << 'L';
       mangle(VD);
       Out << 'E';
     } else if (auto *E = B.dyn_cast<const Expr*>()) {
       NotPrimaryExpr();
       mangleExpression(E);
     } else if (auto TI = B.dyn_cast<TypeInfoLValue>()) {
       NotPrimaryExpr();
       Out << "ti";
       mangleType(QualType(TI.getType(), 0));
     } else {
       // We should never see dynamic allocations here.
       llvm_unreachable("unexpected lvalue base kind in template argument");
     }
 
     switch (Kind) {
     case Base:
       break;
 
     case Offset:
       Out << 'L';
       mangleType(Ctx.getPointerDiffType());
       mangleNumber(V.getLValueOffset().getQuantity());
       Out << 'E';
       break;
 
     case Path:
       // <expression> ::= so <referent type> <expr> [<offset number>]
       //                  <union-selector>* [p] E
       if (!V.getLValueOffset().isZero())
         mangleNumber(V.getLValueOffset().getQuantity());
 
       // We model a past-the-end array pointer as array indexing with index N,
       // not with the "past the end" flag. Compensate for that.
       bool OnePastTheEnd = V.isLValueOnePastTheEnd();
 
       for (APValue::LValuePathEntry E : V.getLValuePath()) {
         if (auto *AT = TypeSoFar->getAsArrayTypeUnsafe()) {
           if (auto *CAT = dyn_cast<ConstantArrayType>(AT))
             OnePastTheEnd |= CAT->getSize() == E.getAsArrayIndex();
           TypeSoFar = AT->getElementType();
         } else {
           const Decl *D = E.getAsBaseOrMember().getPointer();
           if (auto *FD = dyn_cast<FieldDecl>(D)) {
             // <union-selector> ::= _ <number>
             if (FD->getParent()->isUnion()) {
               Out << '_';
               if (FD->getFieldIndex())
                 Out << (FD->getFieldIndex() - 1);
             }
             TypeSoFar = FD->getType();
           } else {
             TypeSoFar = Ctx.getRecordType(cast<CXXRecordDecl>(D));
           }
         }
       }
 
       if (OnePastTheEnd)
         Out << 'p';
       Out << 'E';
       break;
     }
 
     break;
   }
 
   case APValue::MemberPointer:
     // Proposed in https://github.com/itanium-cxx-abi/cxx-abi/issues/47.
     if (!V.getMemberPointerDecl()) {
       mangleNullPointer(T);
       break;
     }
 
     ASTContext &Ctx = Context.getASTContext();
 
     NotPrimaryExpr();
     if (!V.getMemberPointerPath().empty()) {
       Out << "mc";
       mangleType(T);
     } else if (NeedExactType &&
                !Ctx.hasSameType(
                    T->castAs<MemberPointerType>()->getPointeeType(),
                    V.getMemberPointerDecl()->getType()) &&
                Ctx.getLangOpts().getClangABICompat() >
                    LangOptions::ClangABI::Ver11) {
       Out << "cv";
       mangleType(T);
     }
     Out << "adL";
     mangle(V.getMemberPointerDecl());
     Out << 'E';
     if (!V.getMemberPointerPath().empty()) {
       CharUnits Offset =
           Context.getASTContext().getMemberPointerPathAdjustment(V);
       if (!Offset.isZero())
         mangleNumber(Offset.getQuantity());
       Out << 'E';
     }
     break;
   }
 
   if (TopLevel && !IsPrimaryExpr)
     Out << 'E';
 }
 
 void CXXNameMangler::mangleTemplateParameter(unsigned Depth, unsigned Index) {
   // <template-param> ::= T_    # first template parameter
   //                  ::= T <parameter-2 non-negative number> _
   //                  ::= TL <L-1 non-negative number> __
   //                  ::= TL <L-1 non-negative number> _
   //                         <parameter-2 non-negative number> _
   //
   // The latter two manglings are from a proposal here:
   // https://github.com/itanium-cxx-abi/cxx-abi/issues/31#issuecomment-528122117
   Out << 'T';
   if (Depth != 0)
     Out << 'L' << (Depth - 1) << '_';
   if (Index != 0)
     Out << (Index - 1);
   Out << '_';
 }
 
 void CXXNameMangler::mangleSeqID(unsigned SeqID) {
   if (SeqID == 0) {
     // Nothing.
   } else if (SeqID == 1) {
     Out << '0';
   } else {
     SeqID--;
 
     // <seq-id> is encoded in base-36, using digits and upper case letters.
     char Buffer[7]; // log(2**32) / log(36) ~= 7
     MutableArrayRef<char> BufferRef(Buffer);
     MutableArrayRef<char>::reverse_iterator I = BufferRef.rbegin();
 
     for (; SeqID != 0; SeqID /= 36) {
       unsigned C = SeqID % 36;
       *I++ = (C < 10 ? '0' + C : 'A' + C - 10);
     }
 
     Out.write(I.base(), I - BufferRef.rbegin());
   }
   Out << '_';
 }
 
 void CXXNameMangler::mangleExistingSubstitution(TemplateName tname) {
   bool result = mangleSubstitution(tname);
   assert(result && "no existing substitution for template name");
   (void) result;
 }
 
 // <substitution> ::= S <seq-id> _
 //                ::= S_
 bool CXXNameMangler::mangleSubstitution(const NamedDecl *ND) {
   // Try one of the standard substitutions first.
   if (mangleStandardSubstitution(ND))
     return true;
 
   ND = cast<NamedDecl>(ND->getCanonicalDecl());
   return mangleSubstitution(reinterpret_cast<uintptr_t>(ND));
 }
 
 /// Determine whether the given type has any qualifiers that are relevant for
 /// substitutions.
 static bool hasMangledSubstitutionQualifiers(QualType T) {
   Qualifiers Qs = T.getQualifiers();
   return Qs.getCVRQualifiers() || Qs.hasAddressSpace() || Qs.hasUnaligned();
 }
 
 bool CXXNameMangler::mangleSubstitution(QualType T) {
   if (!hasMangledSubstitutionQualifiers(T)) {
     if (const RecordType *RT = T->getAs<RecordType>())
       return mangleSubstitution(RT->getDecl());
   }
 
   uintptr_t TypePtr = reinterpret_cast<uintptr_t>(T.getAsOpaquePtr());
 
   return mangleSubstitution(TypePtr);
 }
 
 bool CXXNameMangler::mangleSubstitution(TemplateName Template) {
   if (TemplateDecl *TD = Template.getAsTemplateDecl())
     return mangleSubstitution(TD);
 
   Template = Context.getASTContext().getCanonicalTemplateName(Template);
   return mangleSubstitution(
                       reinterpret_cast<uintptr_t>(Template.getAsVoidPointer()));
 }
 
 bool CXXNameMangler::mangleSubstitution(uintptr_t Ptr) {
   llvm::DenseMap<uintptr_t, unsigned>::iterator I = Substitutions.find(Ptr);
   if (I == Substitutions.end())
     return false;
 
   unsigned SeqID = I->second;
   Out << 'S';
   mangleSeqID(SeqID);
 
   return true;
 }
 
-static bool isCharType(QualType T) {
-  if (T.isNull())
+/// Returns whether S is a template specialization of std::Name with a single
+/// argument of type A.
+bool CXXNameMangler::isSpecializedAs(QualType S, llvm::StringRef Name,
+                                     QualType A) {
+  if (S.isNull())
     return false;
 
-  return T->isSpecificBuiltinType(BuiltinType::Char_S) ||
-    T->isSpecificBuiltinType(BuiltinType::Char_U);
-}
-
-/// Returns whether a given type is a template specialization of a given name
-/// with a single argument of type char.
-static bool isCharSpecialization(QualType T, const char *Name) {
-  if (T.isNull())
-    return false;
-
-  const RecordType *RT = T->getAs<RecordType>();
+  const RecordType *RT = S->getAs<RecordType>();
   if (!RT)
     return false;
 
   const ClassTemplateSpecializationDecl *SD =
     dyn_cast<ClassTemplateSpecializationDecl>(RT->getDecl());
-  if (!SD)
+  if (!SD || !SD->getIdentifier()->isStr(Name))
     return false;
 
-  if (!isStdNamespace(getEffectiveDeclContext(SD)))
+  if (!isStdNamespace(Context.getEffectiveDeclContext(SD)))
     return false;
 
   const TemplateArgumentList &TemplateArgs = SD->getTemplateArgs();
   if (TemplateArgs.size() != 1)
     return false;
 
-  if (!isCharType(TemplateArgs[0].getAsType()))
+  if (TemplateArgs[0].getAsType() != A)
     return false;
 
-  return SD->getIdentifier()->getName() == Name;
+  return true;
 }
 
-template <std::size_t StrLen>
-static bool isStreamCharSpecialization(const ClassTemplateSpecializationDecl*SD,
-                                       const char (&Str)[StrLen]) {
-  if (!SD->getIdentifier()->isStr(Str))
+/// Returns whether SD is a template specialization std::Name<char,
+/// std::char_traits<char> [, std::allocator<char>]>
+/// HasAllocator controls whether the 3rd template argument is needed.
+bool CXXNameMangler::isStdCharSpecialization(
+    const ClassTemplateSpecializationDecl *SD, llvm::StringRef Name,
+    bool HasAllocator) {
+  if (!SD->getIdentifier()->isStr(Name))
     return false;
 
   const TemplateArgumentList &TemplateArgs = SD->getTemplateArgs();
-  if (TemplateArgs.size() != 2)
+  if (TemplateArgs.size() != (HasAllocator ? 3 : 2))
+    return false;
+
+  QualType A = TemplateArgs[0].getAsType();
+  if (A.isNull())
+    return false;
+  // Plain 'char' is named Char_S or Char_U depending on the target ABI.
+  if (!A->isSpecificBuiltinType(BuiltinType::Char_S) &&
+      !A->isSpecificBuiltinType(BuiltinType::Char_U))
     return false;
 
-  if (!isCharType(TemplateArgs[0].getAsType()))
+  if (!isSpecializedAs(TemplateArgs[1].getAsType(), "char_traits", A))
     return false;
 
-  if (!isCharSpecialization(TemplateArgs[1].getAsType(), "char_traits"))
+  if (HasAllocator &&
+      !isSpecializedAs(TemplateArgs[2].getAsType(), "allocator", A))
     return false;
 
   return true;
 }
 
 bool CXXNameMangler::mangleStandardSubstitution(const NamedDecl *ND) {
   // <substitution> ::= St # ::std::
   if (const NamespaceDecl *NS = dyn_cast<NamespaceDecl>(ND)) {
     if (isStd(NS)) {
       Out << "St";
       return true;
     }
+    return false;
   }
 
   if (const ClassTemplateDecl *TD = dyn_cast<ClassTemplateDecl>(ND)) {
-    if (!isStdNamespace(getEffectiveDeclContext(TD)))
+    if (!isStdNamespace(Context.getEffectiveDeclContext(TD)))
       return false;
 
     // <substitution> ::= Sa # ::std::allocator
     if (TD->getIdentifier()->isStr("allocator")) {
       Out << "Sa";
       return true;
     }
 
     // <<substitution> ::= Sb # ::std::basic_string
     if (TD->getIdentifier()->isStr("basic_string")) {
       Out << "Sb";
       return true;
     }
+    return false;
   }
 
   if (const ClassTemplateSpecializationDecl *SD =
         dyn_cast<ClassTemplateSpecializationDecl>(ND)) {
-    if (!isStdNamespace(getEffectiveDeclContext(SD)))
+    if (!isStdNamespace(Context.getEffectiveDeclContext(SD)))
       return false;
 
     //    <substitution> ::= Ss # ::std::basic_string<char,
     //                            ::std::char_traits<char>,
     //                            ::std::allocator<char> >
-    if (SD->getIdentifier()->isStr("basic_string")) {
-      const TemplateArgumentList &TemplateArgs = SD->getTemplateArgs();
-
-      if (TemplateArgs.size() != 3)
-        return false;
-
-      if (!isCharType(TemplateArgs[0].getAsType()))
-        return false;
-
-      if (!isCharSpecialization(TemplateArgs[1].getAsType(), "char_traits"))
-        return false;
-
-      if (!isCharSpecialization(TemplateArgs[2].getAsType(), "allocator"))
-        return false;
-
+    if (isStdCharSpecialization(SD, "basic_string", /*HasAllocator=*/true)) {
       Out << "Ss";
       return true;
     }
 
     //    <substitution> ::= Si # ::std::basic_istream<char,
     //                            ::std::char_traits<char> >
-    if (isStreamCharSpecialization(SD, "basic_istream")) {
+    if (isStdCharSpecialization(SD, "basic_istream", /*HasAllocator=*/false)) {
       Out << "Si";
       return true;
     }
 
     //    <substitution> ::= So # ::std::basic_ostream<char,
     //                            ::std::char_traits<char> >
-    if (isStreamCharSpecialization(SD, "basic_ostream")) {
+    if (isStdCharSpecialization(SD, "basic_ostream", /*HasAllocator=*/false)) {
       Out << "So";
       return true;
     }
 
     //    <substitution> ::= Sd # ::std::basic_iostream<char,
     //                            ::std::char_traits<char> >
-    if (isStreamCharSpecialization(SD, "basic_iostream")) {
+    if (isStdCharSpecialization(SD, "basic_iostream", /*HasAllocator=*/false)) {
       Out << "Sd";
       return true;
     }
+    return false;
   }
+
   return false;
 }
 
 void CXXNameMangler::addSubstitution(QualType T) {
   if (!hasMangledSubstitutionQualifiers(T)) {
     if (const RecordType *RT = T->getAs<RecordType>()) {
       addSubstitution(RT->getDecl());
       return;
     }
   }
 
   uintptr_t TypePtr = reinterpret_cast<uintptr_t>(T.getAsOpaquePtr());
   addSubstitution(TypePtr);
 }
 
 void CXXNameMangler::addSubstitution(TemplateName Template) {
   if (TemplateDecl *TD = Template.getAsTemplateDecl())
     return addSubstitution(TD);
 
   Template = Context.getASTContext().getCanonicalTemplateName(Template);
   addSubstitution(reinterpret_cast<uintptr_t>(Template.getAsVoidPointer()));
 }
 
 void CXXNameMangler::addSubstitution(uintptr_t Ptr) {
   assert(!Substitutions.count(Ptr) && "Substitution already exists!");
   Substitutions[Ptr] = SeqID++;
 }
 
 void CXXNameMangler::extendSubstitutions(CXXNameMangler* Other) {
   assert(Other->SeqID >= SeqID && "Must be superset of substitutions!");
   if (Other->SeqID > SeqID) {
     Substitutions.swap(Other->Substitutions);
     SeqID = Other->SeqID;
   }
 }
 
 CXXNameMangler::AbiTagList
 CXXNameMangler::makeFunctionReturnTypeTags(const FunctionDecl *FD) {
   // When derived abi tags are disabled there is no need to make any list.
   if (DisableDerivedAbiTags)
     return AbiTagList();
 
   llvm::raw_null_ostream NullOutStream;
   CXXNameMangler TrackReturnTypeTags(*this, NullOutStream);
   TrackReturnTypeTags.disableDerivedAbiTags();
 
   const FunctionProtoType *Proto =
       cast<FunctionProtoType>(FD->getType()->getAs<FunctionType>());
   FunctionTypeDepthState saved = TrackReturnTypeTags.FunctionTypeDepth.push();
   TrackReturnTypeTags.FunctionTypeDepth.enterResultType();
   TrackReturnTypeTags.mangleType(Proto->getReturnType());
   TrackReturnTypeTags.FunctionTypeDepth.leaveResultType();
   TrackReturnTypeTags.FunctionTypeDepth.pop(saved);
 
   return TrackReturnTypeTags.AbiTagsRoot.getSortedUniqueUsedAbiTags();
 }
 
 CXXNameMangler::AbiTagList
 CXXNameMangler::makeVariableTypeTags(const VarDecl *VD) {
   // When derived abi tags are disabled there is no need to make any list.
   if (DisableDerivedAbiTags)
     return AbiTagList();
 
   llvm::raw_null_ostream NullOutStream;
   CXXNameMangler TrackVariableType(*this, NullOutStream);
   TrackVariableType.disableDerivedAbiTags();
 
   TrackVariableType.mangleType(VD->getType());
 
   return TrackVariableType.AbiTagsRoot.getSortedUniqueUsedAbiTags();
 }
 
 bool CXXNameMangler::shouldHaveAbiTags(ItaniumMangleContextImpl &C,
                                        const VarDecl *VD) {
   llvm::raw_null_ostream NullOutStream;
   CXXNameMangler TrackAbiTags(C, NullOutStream, nullptr, true);
   TrackAbiTags.mangle(VD);
   return TrackAbiTags.AbiTagsRoot.getUsedAbiTags().size();
 }
 
 //
 
 /// Mangles the name of the declaration D and emits that name to the given
 /// output stream.
 ///
 /// If the declaration D requires a mangled name, this routine will emit that
 /// mangled name to \p os and return true. Otherwise, \p os will be unchanged
 /// and this routine will return false. In this case, the caller should just
 /// emit the identifier of the declaration (\c D->getIdentifier()) as its
 /// name.
 void ItaniumMangleContextImpl::mangleCXXName(GlobalDecl GD,
                                              raw_ostream &Out) {
   const NamedDecl *D = cast<NamedDecl>(GD.getDecl());
   assert((isa<FunctionDecl, VarDecl, TemplateParamObjectDecl>(D)) &&
          "Invalid mangleName() call, argument is not a variable or function!");
 
   PrettyStackTraceDecl CrashInfo(D, SourceLocation(),
                                  getASTContext().getSourceManager(),
                                  "Mangling declaration");
 
   if (auto *CD = dyn_cast<CXXConstructorDecl>(D)) {
     auto Type = GD.getCtorType();
     CXXNameMangler Mangler(*this, Out, CD, Type);
     return Mangler.mangle(GlobalDecl(CD, Type));
   }
 
   if (auto *DD = dyn_cast<CXXDestructorDecl>(D)) {
     auto Type = GD.getDtorType();
     CXXNameMangler Mangler(*this, Out, DD, Type);
     return Mangler.mangle(GlobalDecl(DD, Type));
   }
 
   CXXNameMangler Mangler(*this, Out, D);
   Mangler.mangle(GD);
 }
 
 void ItaniumMangleContextImpl::mangleCXXCtorComdat(const CXXConstructorDecl *D,
                                                    raw_ostream &Out) {
   CXXNameMangler Mangler(*this, Out, D, Ctor_Comdat);
   Mangler.mangle(GlobalDecl(D, Ctor_Comdat));
 }
 
 void ItaniumMangleContextImpl::mangleCXXDtorComdat(const CXXDestructorDecl *D,
                                                    raw_ostream &Out) {
   CXXNameMangler Mangler(*this, Out, D, Dtor_Comdat);
   Mangler.mangle(GlobalDecl(D, Dtor_Comdat));
 }
 
 void ItaniumMangleContextImpl::mangleThunk(const CXXMethodDecl *MD,
                                            const ThunkInfo &Thunk,
                                            raw_ostream &Out) {
   //  <special-name> ::= T <call-offset> <base encoding>
   //                      # base is the nominal target function of thunk
   //  <special-name> ::= Tc <call-offset> <call-offset> <base encoding>
   //                      # base is the nominal target function of thunk
   //                      # first call-offset is 'this' adjustment
   //                      # second call-offset is result adjustment
 
   assert(!isa<CXXDestructorDecl>(MD) &&
          "Use mangleCXXDtor for destructor decls!");
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZT";
   if (!Thunk.Return.isEmpty())
     Mangler.getStream() << 'c';
 
   // Mangle the 'this' pointer adjustment.
   Mangler.mangleCallOffset(Thunk.This.NonVirtual,
                            Thunk.This.Virtual.Itanium.VCallOffsetOffset);
 
   // Mangle the return pointer adjustment if there is one.
   if (!Thunk.Return.isEmpty())
     Mangler.mangleCallOffset(Thunk.Return.NonVirtual,
                              Thunk.Return.Virtual.Itanium.VBaseOffsetOffset);
 
   Mangler.mangleFunctionEncoding(MD);
 }
 
 void ItaniumMangleContextImpl::mangleCXXDtorThunk(
     const CXXDestructorDecl *DD, CXXDtorType Type,
     const ThisAdjustment &ThisAdjustment, raw_ostream &Out) {
   //  <special-name> ::= T <call-offset> <base encoding>
   //                      # base is the nominal target function of thunk
   CXXNameMangler Mangler(*this, Out, DD, Type);
   Mangler.getStream() << "_ZT";
 
   // Mangle the 'this' pointer adjustment.
   Mangler.mangleCallOffset(ThisAdjustment.NonVirtual,
                            ThisAdjustment.Virtual.Itanium.VCallOffsetOffset);
 
   Mangler.mangleFunctionEncoding(GlobalDecl(DD, Type));
 }
 
 /// Returns the mangled name for a guard variable for the passed in VarDecl.
 void ItaniumMangleContextImpl::mangleStaticGuardVariable(const VarDecl *D,
                                                          raw_ostream &Out) {
   //  <special-name> ::= GV <object name>       # Guard variable for one-time
   //                                            # initialization
   CXXNameMangler Mangler(*this, Out);
   // GCC 5.3.0 doesn't emit derived ABI tags for local names but that seems to
   // be a bug that is fixed in trunk.
   Mangler.getStream() << "_ZGV";
   Mangler.mangleName(D);
 }
 
 void ItaniumMangleContextImpl::mangleDynamicInitializer(const VarDecl *MD,
                                                         raw_ostream &Out) {
   // These symbols are internal in the Itanium ABI, so the names don't matter.
   // Clang has traditionally used this symbol and allowed LLVM to adjust it to
   // avoid duplicate symbols.
   Out << "__cxx_global_var_init";
 }
 
 void ItaniumMangleContextImpl::mangleDynamicAtExitDestructor(const VarDecl *D,
                                                              raw_ostream &Out) {
   // Prefix the mangling of D with __dtor_.
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "__dtor_";
   if (shouldMangleDeclName(D))
     Mangler.mangle(D);
   else
     Mangler.getStream() << D->getName();
 }
 
 void ItaniumMangleContextImpl::mangleDynamicStermFinalizer(const VarDecl *D,
                                                            raw_ostream &Out) {
   // Clang generates these internal-linkage functions as part of its
   // implementation of the XL ABI.
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "__finalize_";
   if (shouldMangleDeclName(D))
     Mangler.mangle(D);
   else
     Mangler.getStream() << D->getName();
 }
 
 void ItaniumMangleContextImpl::mangleSEHFilterExpression(
     const NamedDecl *EnclosingDecl, raw_ostream &Out) {
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "__filt_";
   if (shouldMangleDeclName(EnclosingDecl))
     Mangler.mangle(EnclosingDecl);
   else
     Mangler.getStream() << EnclosingDecl->getName();
 }
 
 void ItaniumMangleContextImpl::mangleSEHFinallyBlock(
     const NamedDecl *EnclosingDecl, raw_ostream &Out) {
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "__fin_";
   if (shouldMangleDeclName(EnclosingDecl))
     Mangler.mangle(EnclosingDecl);
   else
     Mangler.getStream() << EnclosingDecl->getName();
 }
 
 void ItaniumMangleContextImpl::mangleItaniumThreadLocalInit(const VarDecl *D,
                                                             raw_ostream &Out) {
   //  <special-name> ::= TH <object name>
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTH";
   Mangler.mangleName(D);
 }
 
 void
 ItaniumMangleContextImpl::mangleItaniumThreadLocalWrapper(const VarDecl *D,
                                                           raw_ostream &Out) {
   //  <special-name> ::= TW <object name>
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTW";
   Mangler.mangleName(D);
 }
 
 void ItaniumMangleContextImpl::mangleReferenceTemporary(const VarDecl *D,
                                                         unsigned ManglingNumber,
                                                         raw_ostream &Out) {
   // We match the GCC mangling here.
   //  <special-name> ::= GR <object name>
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZGR";
   Mangler.mangleName(D);
   assert(ManglingNumber > 0 && "Reference temporary mangling number is zero!");
   Mangler.mangleSeqID(ManglingNumber - 1);
 }
 
 void ItaniumMangleContextImpl::mangleCXXVTable(const CXXRecordDecl *RD,
                                                raw_ostream &Out) {
   // <special-name> ::= TV <type>  # virtual table
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTV";
   Mangler.mangleNameOrStandardSubstitution(RD);
 }
 
 void ItaniumMangleContextImpl::mangleCXXVTT(const CXXRecordDecl *RD,
                                             raw_ostream &Out) {
   // <special-name> ::= TT <type>  # VTT structure
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTT";
   Mangler.mangleNameOrStandardSubstitution(RD);
 }
 
 void ItaniumMangleContextImpl::mangleCXXCtorVTable(const CXXRecordDecl *RD,
                                                    int64_t Offset,
                                                    const CXXRecordDecl *Type,
                                                    raw_ostream &Out) {
   // <special-name> ::= TC <type> <offset number> _ <base type>
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTC";
   Mangler.mangleNameOrStandardSubstitution(RD);
   Mangler.getStream() << Offset;
   Mangler.getStream() << '_';
   Mangler.mangleNameOrStandardSubstitution(Type);
 }
 
 void ItaniumMangleContextImpl::mangleCXXRTTI(QualType Ty, raw_ostream &Out) {
   // <special-name> ::= TI <type>  # typeinfo structure
   assert(!Ty.hasQualifiers() && "RTTI info cannot have top-level qualifiers");
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTI";
   Mangler.mangleType(Ty);
 }
 
 void ItaniumMangleContextImpl::mangleCXXRTTIName(QualType Ty,
                                                  raw_ostream &Out) {
   // <special-name> ::= TS <type>  # typeinfo name (null terminated byte string)
   CXXNameMangler Mangler(*this, Out);
   Mangler.getStream() << "_ZTS";
   Mangler.mangleType(Ty);
 }
 
 void ItaniumMangleContextImpl::mangleTypeName(QualType Ty, raw_ostream &Out) {
   mangleCXXRTTIName(Ty, Out);
 }
 
 void ItaniumMangleContextImpl::mangleStringLiteral(const StringLiteral *, raw_ostream &) {
   llvm_unreachable("Can't mangle string literals");
 }
 
 void ItaniumMangleContextImpl::mangleLambdaSig(const CXXRecordDecl *Lambda,
                                                raw_ostream &Out) {
   CXXNameMangler Mangler(*this, Out);
   Mangler.mangleLambdaSig(Lambda);
 }
 
 ItaniumMangleContext *ItaniumMangleContext::create(ASTContext &Context,
                                                    DiagnosticsEngine &Diags) {
   return new ItaniumMangleContextImpl(
       Context, Diags,
       [](ASTContext &, const NamedDecl *) -> llvm::Optional<unsigned> {
         return llvm::None;
       });
 }
 
 ItaniumMangleContext *
 ItaniumMangleContext::create(ASTContext &Context, DiagnosticsEngine &Diags,
                              DiscriminatorOverrideTy DiscriminatorOverride) {
   return new ItaniumMangleContextImpl(Context, Diags, DiscriminatorOverride);
 }
diff --git a/contrib/llvm-project/clang/lib/AST/TypePrinter.cpp b/contrib/llvm-project/clang/lib/AST/TypePrinter.cpp
index bba323f651aa..6e827530f41b 100644
--- a/contrib/llvm-project/clang/lib/AST/TypePrinter.cpp
+++ b/contrib/llvm-project/clang/lib/AST/TypePrinter.cpp
@@ -1,2297 +1,2296 @@
 //===- TypePrinter.cpp - Pretty-Print Clang Types -------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This contains code to print types from Clang's type system.
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclBase.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/NestedNameSpecifier.h"
 #include "clang/AST/PrettyPrinter.h"
 #include "clang/AST/TemplateBase.h"
 #include "clang/AST/TemplateName.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/AddressSpaces.h"
 #include "clang/Basic/ExceptionSpecificationType.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/Specifiers.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/SaveAndRestore.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <string>
 
 using namespace clang;
 
 namespace {
 
   /// RAII object that enables printing of the ARC __strong lifetime
   /// qualifier.
   class IncludeStrongLifetimeRAII {
     PrintingPolicy &Policy;
     bool Old;
 
   public:
     explicit IncludeStrongLifetimeRAII(PrintingPolicy &Policy)
         : Policy(Policy), Old(Policy.SuppressStrongLifetime) {
         if (!Policy.SuppressLifetimeQualifiers)
           Policy.SuppressStrongLifetime = false;
     }
 
     ~IncludeStrongLifetimeRAII() {
       Policy.SuppressStrongLifetime = Old;
     }
   };
 
   class ParamPolicyRAII {
     PrintingPolicy &Policy;
     bool Old;
 
   public:
     explicit ParamPolicyRAII(PrintingPolicy &Policy)
         : Policy(Policy), Old(Policy.SuppressSpecifiers) {
       Policy.SuppressSpecifiers = false;
     }
 
     ~ParamPolicyRAII() {
       Policy.SuppressSpecifiers = Old;
     }
   };
 
   class ElaboratedTypePolicyRAII {
     PrintingPolicy &Policy;
     bool SuppressTagKeyword;
     bool SuppressScope;
 
   public:
     explicit ElaboratedTypePolicyRAII(PrintingPolicy &Policy) : Policy(Policy) {
       SuppressTagKeyword = Policy.SuppressTagKeyword;
       SuppressScope = Policy.SuppressScope;
       Policy.SuppressTagKeyword = true;
       Policy.SuppressScope = true;
     }
 
     ~ElaboratedTypePolicyRAII() {
       Policy.SuppressTagKeyword = SuppressTagKeyword;
       Policy.SuppressScope = SuppressScope;
     }
   };
 
   class TypePrinter {
     PrintingPolicy Policy;
     unsigned Indentation;
     bool HasEmptyPlaceHolder = false;
     bool InsideCCAttribute = false;
 
   public:
     explicit TypePrinter(const PrintingPolicy &Policy, unsigned Indentation = 0)
         : Policy(Policy), Indentation(Indentation) {}
 
     void print(const Type *ty, Qualifiers qs, raw_ostream &OS,
                StringRef PlaceHolder);
     void print(QualType T, raw_ostream &OS, StringRef PlaceHolder);
 
     static bool canPrefixQualifiers(const Type *T, bool &NeedARCStrongQualifier);
     void spaceBeforePlaceHolder(raw_ostream &OS);
     void printTypeSpec(NamedDecl *D, raw_ostream &OS);
     void printTemplateId(const TemplateSpecializationType *T, raw_ostream &OS,
                          bool FullyQualify);
 
     void printBefore(QualType T, raw_ostream &OS);
     void printAfter(QualType T, raw_ostream &OS);
     void AppendScope(DeclContext *DC, raw_ostream &OS,
                      DeclarationName NameInScope);
     void printTag(TagDecl *T, raw_ostream &OS);
     void printFunctionAfter(const FunctionType::ExtInfo &Info, raw_ostream &OS);
 #define ABSTRACT_TYPE(CLASS, PARENT)
 #define TYPE(CLASS, PARENT) \
     void print##CLASS##Before(const CLASS##Type *T, raw_ostream &OS); \
     void print##CLASS##After(const CLASS##Type *T, raw_ostream &OS);
 #include "clang/AST/TypeNodes.inc"
 
   private:
     void printBefore(const Type *ty, Qualifiers qs, raw_ostream &OS);
     void printAfter(const Type *ty, Qualifiers qs, raw_ostream &OS);
   };
 
 } // namespace
 
 static void AppendTypeQualList(raw_ostream &OS, unsigned TypeQuals,
                                bool HasRestrictKeyword) {
   bool appendSpace = false;
   if (TypeQuals & Qualifiers::Const) {
     OS << "const";
     appendSpace = true;
   }
   if (TypeQuals & Qualifiers::Volatile) {
     if (appendSpace) OS << ' ';
     OS << "volatile";
     appendSpace = true;
   }
   if (TypeQuals & Qualifiers::Restrict) {
     if (appendSpace) OS << ' ';
     if (HasRestrictKeyword) {
       OS << "restrict";
     } else {
       OS << "__restrict";
     }
   }
 }
 
 void TypePrinter::spaceBeforePlaceHolder(raw_ostream &OS) {
   if (!HasEmptyPlaceHolder)
     OS << ' ';
 }
 
 static SplitQualType splitAccordingToPolicy(QualType QT,
                                             const PrintingPolicy &Policy) {
   if (Policy.PrintCanonicalTypes)
     QT = QT.getCanonicalType();
   return QT.split();
 }
 
 void TypePrinter::print(QualType t, raw_ostream &OS, StringRef PlaceHolder) {
   SplitQualType split = splitAccordingToPolicy(t, Policy);
   print(split.Ty, split.Quals, OS, PlaceHolder);
 }
 
 void TypePrinter::print(const Type *T, Qualifiers Quals, raw_ostream &OS,
                         StringRef PlaceHolder) {
   if (!T) {
     OS << "NULL TYPE";
     return;
   }
 
   SaveAndRestore<bool> PHVal(HasEmptyPlaceHolder, PlaceHolder.empty());
 
   printBefore(T, Quals, OS);
   OS << PlaceHolder;
   printAfter(T, Quals, OS);
 }
 
 bool TypePrinter::canPrefixQualifiers(const Type *T,
                                       bool &NeedARCStrongQualifier) {
   // CanPrefixQualifiers - We prefer to print type qualifiers before the type,
   // so that we get "const int" instead of "int const", but we can't do this if
   // the type is complex.  For example if the type is "int*", we *must* print
   // "int * const", printing "const int *" is different.  Only do this when the
   // type expands to a simple string.
   bool CanPrefixQualifiers = false;
   NeedARCStrongQualifier = false;
   const Type *UnderlyingType = T;
   if (const auto *AT = dyn_cast<AutoType>(T))
     UnderlyingType = AT->desugar().getTypePtr();
   if (const auto *Subst = dyn_cast<SubstTemplateTypeParmType>(T))
     UnderlyingType = Subst->getReplacementType().getTypePtr();
   Type::TypeClass TC = UnderlyingType->getTypeClass();
 
   switch (TC) {
     case Type::Auto:
     case Type::Builtin:
     case Type::Complex:
     case Type::UnresolvedUsing:
     case Type::Using:
     case Type::Typedef:
     case Type::TypeOfExpr:
     case Type::TypeOf:
     case Type::Decltype:
     case Type::UnaryTransform:
     case Type::Record:
     case Type::Enum:
     case Type::Elaborated:
     case Type::TemplateTypeParm:
     case Type::SubstTemplateTypeParmPack:
     case Type::DeducedTemplateSpecialization:
     case Type::TemplateSpecialization:
     case Type::InjectedClassName:
     case Type::DependentName:
     case Type::DependentTemplateSpecialization:
     case Type::ObjCObject:
     case Type::ObjCTypeParam:
     case Type::ObjCInterface:
     case Type::Atomic:
     case Type::Pipe:
     case Type::BitInt:
     case Type::DependentBitInt:
       CanPrefixQualifiers = true;
       break;
 
     case Type::ObjCObjectPointer:
       CanPrefixQualifiers = T->isObjCIdType() || T->isObjCClassType() ||
         T->isObjCQualifiedIdType() || T->isObjCQualifiedClassType();
       break;
 
     case Type::VariableArray:
     case Type::DependentSizedArray:
       NeedARCStrongQualifier = true;
       LLVM_FALLTHROUGH;
 
     case Type::ConstantArray:
     case Type::IncompleteArray:
       return canPrefixQualifiers(
           cast<ArrayType>(UnderlyingType)->getElementType().getTypePtr(),
           NeedARCStrongQualifier);
 
     case Type::Adjusted:
     case Type::Decayed:
     case Type::Pointer:
     case Type::BlockPointer:
     case Type::LValueReference:
     case Type::RValueReference:
     case Type::MemberPointer:
     case Type::DependentAddressSpace:
     case Type::DependentVector:
     case Type::DependentSizedExtVector:
     case Type::Vector:
     case Type::ExtVector:
     case Type::ConstantMatrix:
     case Type::DependentSizedMatrix:
     case Type::FunctionProto:
     case Type::FunctionNoProto:
     case Type::Paren:
     case Type::PackExpansion:
     case Type::SubstTemplateTypeParm:
     case Type::MacroQualified:
       CanPrefixQualifiers = false;
       break;
 
     case Type::Attributed: {
       // We still want to print the address_space before the type if it is an
       // address_space attribute.
       const auto *AttrTy = cast<AttributedType>(UnderlyingType);
       CanPrefixQualifiers = AttrTy->getAttrKind() == attr::AddressSpace;
     }
   }
 
   return CanPrefixQualifiers;
 }
 
 void TypePrinter::printBefore(QualType T, raw_ostream &OS) {
   SplitQualType Split = splitAccordingToPolicy(T, Policy);
 
   // If we have cv1 T, where T is substituted for cv2 U, only print cv1 - cv2
   // at this level.
   Qualifiers Quals = Split.Quals;
   if (const auto *Subst = dyn_cast<SubstTemplateTypeParmType>(Split.Ty))
     Quals -= QualType(Subst, 0).getQualifiers();
 
   printBefore(Split.Ty, Quals, OS);
 }
 
 /// Prints the part of the type string before an identifier, e.g. for
 /// "int foo[10]" it prints "int ".
 void TypePrinter::printBefore(const Type *T,Qualifiers Quals, raw_ostream &OS) {
   if (Policy.SuppressSpecifiers && T->isSpecifierType())
     return;
 
   SaveAndRestore<bool> PrevPHIsEmpty(HasEmptyPlaceHolder);
 
   // Print qualifiers as appropriate.
 
   bool CanPrefixQualifiers = false;
   bool NeedARCStrongQualifier = false;
   CanPrefixQualifiers = canPrefixQualifiers(T, NeedARCStrongQualifier);
 
   if (CanPrefixQualifiers && !Quals.empty()) {
     if (NeedARCStrongQualifier) {
       IncludeStrongLifetimeRAII Strong(Policy);
       Quals.print(OS, Policy, /*appendSpaceIfNonEmpty=*/true);
     } else {
       Quals.print(OS, Policy, /*appendSpaceIfNonEmpty=*/true);
     }
   }
 
   bool hasAfterQuals = false;
   if (!CanPrefixQualifiers && !Quals.empty()) {
     hasAfterQuals = !Quals.isEmptyWhenPrinted(Policy);
     if (hasAfterQuals)
       HasEmptyPlaceHolder = false;
   }
 
   switch (T->getTypeClass()) {
 #define ABSTRACT_TYPE(CLASS, PARENT)
 #define TYPE(CLASS, PARENT) case Type::CLASS: \
     print##CLASS##Before(cast<CLASS##Type>(T), OS); \
     break;
 #include "clang/AST/TypeNodes.inc"
   }
 
   if (hasAfterQuals) {
     if (NeedARCStrongQualifier) {
       IncludeStrongLifetimeRAII Strong(Policy);
       Quals.print(OS, Policy, /*appendSpaceIfNonEmpty=*/!PrevPHIsEmpty.get());
     } else {
       Quals.print(OS, Policy, /*appendSpaceIfNonEmpty=*/!PrevPHIsEmpty.get());
     }
   }
 }
 
 void TypePrinter::printAfter(QualType t, raw_ostream &OS) {
   SplitQualType split = splitAccordingToPolicy(t, Policy);
   printAfter(split.Ty, split.Quals, OS);
 }
 
 /// Prints the part of the type string after an identifier, e.g. for
 /// "int foo[10]" it prints "[10]".
 void TypePrinter::printAfter(const Type *T, Qualifiers Quals, raw_ostream &OS) {
   switch (T->getTypeClass()) {
 #define ABSTRACT_TYPE(CLASS, PARENT)
 #define TYPE(CLASS, PARENT) case Type::CLASS: \
     print##CLASS##After(cast<CLASS##Type>(T), OS); \
     break;
 #include "clang/AST/TypeNodes.inc"
   }
 }
 
 void TypePrinter::printBuiltinBefore(const BuiltinType *T, raw_ostream &OS) {
   OS << T->getName(Policy);
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printBuiltinAfter(const BuiltinType *T, raw_ostream &OS) {}
 
 void TypePrinter::printComplexBefore(const ComplexType *T, raw_ostream &OS) {
   OS << "_Complex ";
   printBefore(T->getElementType(), OS);
 }
 
 void TypePrinter::printComplexAfter(const ComplexType *T, raw_ostream &OS) {
   printAfter(T->getElementType(), OS);
 }
 
 void TypePrinter::printPointerBefore(const PointerType *T, raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   SaveAndRestore<bool> NonEmptyPH(HasEmptyPlaceHolder, false);
   printBefore(T->getPointeeType(), OS);
   // Handle things like 'int (*A)[4];' correctly.
   // FIXME: this should include vectors, but vectors use attributes I guess.
   if (isa<ArrayType>(T->getPointeeType()))
     OS << '(';
   OS << '*';
 }
 
 void TypePrinter::printPointerAfter(const PointerType *T, raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   SaveAndRestore<bool> NonEmptyPH(HasEmptyPlaceHolder, false);
   // Handle things like 'int (*A)[4];' correctly.
   // FIXME: this should include vectors, but vectors use attributes I guess.
   if (isa<ArrayType>(T->getPointeeType()))
     OS << ')';
   printAfter(T->getPointeeType(), OS);
 }
 
 void TypePrinter::printBlockPointerBefore(const BlockPointerType *T,
                                           raw_ostream &OS) {
   SaveAndRestore<bool> NonEmptyPH(HasEmptyPlaceHolder, false);
   printBefore(T->getPointeeType(), OS);
   OS << '^';
 }
 
 void TypePrinter::printBlockPointerAfter(const BlockPointerType *T,
                                           raw_ostream &OS) {
   SaveAndRestore<bool> NonEmptyPH(HasEmptyPlaceHolder, false);
   printAfter(T->getPointeeType(), OS);
 }
 
 // When printing a reference, the referenced type might also be a reference.
 // If so, we want to skip that before printing the inner type.
 static QualType skipTopLevelReferences(QualType T) {
   if (auto *Ref = T->getAs<ReferenceType>())
     return skipTopLevelReferences(Ref->getPointeeTypeAsWritten());
   return T;
 }
 
 void TypePrinter::printLValueReferenceBefore(const LValueReferenceType *T,
                                              raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   SaveAndRestore<bool> NonEmptyPH(HasEmptyPlaceHolder, false);
   QualType Inner = skipTopLevelReferences(T->getPointeeTypeAsWritten());
   printBefore(Inner, OS);
   // Handle things like 'int (&A)[4];' correctly.
   // FIXME: this should include vectors, but vectors use attributes I guess.
   if (isa<ArrayType>(Inner))
     OS << '(';
   OS << '&';
 }
 
 void TypePrinter::printLValueReferenceAfter(const LValueReferenceType *T,
                                             raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   SaveAndRestore<bool> NonEmptyPH(HasEmptyPlaceHolder, false);
   QualType Inner = skipTopLevelReferences(T->getPointeeTypeAsWritten());
   // Handle things like 'int (&A)[4];' correctly.
   // FIXME: this should include vectors, but vectors use attributes I guess.
   if (isa<ArrayType>(Inner))
     OS << ')';
   printAfter(Inner, OS);
 }
 
 void TypePrinter::printRValueReferenceBefore(const RValueReferenceType *T,
                                              raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   SaveAndRestore<bool> NonEmptyPH(HasEmptyPlaceHolder, false);
   QualType Inner = skipTopLevelReferences(T->getPointeeTypeAsWritten());
   printBefore(Inner, OS);
   // Handle things like 'int (&&A)[4];' correctly.
   // FIXME: this should include vectors, but vectors use attributes I guess.
   if (isa<ArrayType>(Inner))
     OS << '(';
   OS << "&&";
 }
 
 void TypePrinter::printRValueReferenceAfter(const RValueReferenceType *T,
                                             raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   SaveAndRestore<bool> NonEmptyPH(HasEmptyPlaceHolder, false);
   QualType Inner = skipTopLevelReferences(T->getPointeeTypeAsWritten());
   // Handle things like 'int (&&A)[4];' correctly.
   // FIXME: this should include vectors, but vectors use attributes I guess.
   if (isa<ArrayType>(Inner))
     OS << ')';
   printAfter(Inner, OS);
 }
 
 void TypePrinter::printMemberPointerBefore(const MemberPointerType *T,
                                            raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   SaveAndRestore<bool> NonEmptyPH(HasEmptyPlaceHolder, false);
   printBefore(T->getPointeeType(), OS);
   // Handle things like 'int (Cls::*A)[4];' correctly.
   // FIXME: this should include vectors, but vectors use attributes I guess.
   if (isa<ArrayType>(T->getPointeeType()))
     OS << '(';
 
   PrintingPolicy InnerPolicy(Policy);
   InnerPolicy.IncludeTagDefinition = false;
   TypePrinter(InnerPolicy).print(QualType(T->getClass(), 0), OS, StringRef());
 
   OS << "::*";
 }
 
 void TypePrinter::printMemberPointerAfter(const MemberPointerType *T,
                                           raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   SaveAndRestore<bool> NonEmptyPH(HasEmptyPlaceHolder, false);
   // Handle things like 'int (Cls::*A)[4];' correctly.
   // FIXME: this should include vectors, but vectors use attributes I guess.
   if (isa<ArrayType>(T->getPointeeType()))
     OS << ')';
   printAfter(T->getPointeeType(), OS);
 }
 
 void TypePrinter::printConstantArrayBefore(const ConstantArrayType *T,
                                            raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   printBefore(T->getElementType(), OS);
 }
 
 void TypePrinter::printConstantArrayAfter(const ConstantArrayType *T,
                                           raw_ostream &OS) {
   OS << '[';
   if (T->getIndexTypeQualifiers().hasQualifiers()) {
     AppendTypeQualList(OS, T->getIndexTypeCVRQualifiers(),
                        Policy.Restrict);
     OS << ' ';
   }
 
   if (T->getSizeModifier() == ArrayType::Static)
     OS << "static ";
 
   OS << T->getSize().getZExtValue() << ']';
   printAfter(T->getElementType(), OS);
 }
 
 void TypePrinter::printIncompleteArrayBefore(const IncompleteArrayType *T,
                                              raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   printBefore(T->getElementType(), OS);
 }
 
 void TypePrinter::printIncompleteArrayAfter(const IncompleteArrayType *T,
                                             raw_ostream &OS) {
   OS << "[]";
   printAfter(T->getElementType(), OS);
 }
 
 void TypePrinter::printVariableArrayBefore(const VariableArrayType *T,
                                            raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   printBefore(T->getElementType(), OS);
 }
 
 void TypePrinter::printVariableArrayAfter(const VariableArrayType *T,
                                           raw_ostream &OS) {
   OS << '[';
   if (T->getIndexTypeQualifiers().hasQualifiers()) {
     AppendTypeQualList(OS, T->getIndexTypeCVRQualifiers(), Policy.Restrict);
     OS << ' ';
   }
 
   if (T->getSizeModifier() == VariableArrayType::Static)
     OS << "static ";
   else if (T->getSizeModifier() == VariableArrayType::Star)
     OS << '*';
 
   if (T->getSizeExpr())
     T->getSizeExpr()->printPretty(OS, nullptr, Policy);
   OS << ']';
 
   printAfter(T->getElementType(), OS);
 }
 
 void TypePrinter::printAdjustedBefore(const AdjustedType *T, raw_ostream &OS) {
   // Print the adjusted representation, otherwise the adjustment will be
   // invisible.
   printBefore(T->getAdjustedType(), OS);
 }
 
 void TypePrinter::printAdjustedAfter(const AdjustedType *T, raw_ostream &OS) {
   printAfter(T->getAdjustedType(), OS);
 }
 
 void TypePrinter::printDecayedBefore(const DecayedType *T, raw_ostream &OS) {
   // Print as though it's a pointer.
   printAdjustedBefore(T, OS);
 }
 
 void TypePrinter::printDecayedAfter(const DecayedType *T, raw_ostream &OS) {
   printAdjustedAfter(T, OS);
 }
 
 void TypePrinter::printDependentSizedArrayBefore(
                                                const DependentSizedArrayType *T,
                                                raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   printBefore(T->getElementType(), OS);
 }
 
 void TypePrinter::printDependentSizedArrayAfter(
                                                const DependentSizedArrayType *T,
                                                raw_ostream &OS) {
   OS << '[';
   if (T->getSizeExpr())
     T->getSizeExpr()->printPretty(OS, nullptr, Policy);
   OS << ']';
   printAfter(T->getElementType(), OS);
 }
 
 void TypePrinter::printDependentAddressSpaceBefore(
     const DependentAddressSpaceType *T, raw_ostream &OS) {
   printBefore(T->getPointeeType(), OS);
 }
 
 void TypePrinter::printDependentAddressSpaceAfter(
     const DependentAddressSpaceType *T, raw_ostream &OS) {
   OS << " __attribute__((address_space(";
   if (T->getAddrSpaceExpr())
     T->getAddrSpaceExpr()->printPretty(OS, nullptr, Policy);
   OS << ")))";
   printAfter(T->getPointeeType(), OS);
 }
 
 void TypePrinter::printDependentSizedExtVectorBefore(
                                           const DependentSizedExtVectorType *T,
                                           raw_ostream &OS) {
   printBefore(T->getElementType(), OS);
 }
 
 void TypePrinter::printDependentSizedExtVectorAfter(
                                           const DependentSizedExtVectorType *T,
                                           raw_ostream &OS) {
   OS << " __attribute__((ext_vector_type(";
   if (T->getSizeExpr())
     T->getSizeExpr()->printPretty(OS, nullptr, Policy);
   OS << ")))";
   printAfter(T->getElementType(), OS);
 }
 
 void TypePrinter::printVectorBefore(const VectorType *T, raw_ostream &OS) {
   switch (T->getVectorKind()) {
   case VectorType::AltiVecPixel:
     OS << "__vector __pixel ";
     break;
   case VectorType::AltiVecBool:
     OS << "__vector __bool ";
     printBefore(T->getElementType(), OS);
     break;
   case VectorType::AltiVecVector:
     OS << "__vector ";
     printBefore(T->getElementType(), OS);
     break;
   case VectorType::NeonVector:
     OS << "__attribute__((neon_vector_type("
        << T->getNumElements() << "))) ";
     printBefore(T->getElementType(), OS);
     break;
   case VectorType::NeonPolyVector:
     OS << "__attribute__((neon_polyvector_type(" <<
           T->getNumElements() << "))) ";
     printBefore(T->getElementType(), OS);
     break;
   case VectorType::GenericVector: {
     // FIXME: We prefer to print the size directly here, but have no way
     // to get the size of the type.
     OS << "__attribute__((__vector_size__("
        << T->getNumElements()
        << " * sizeof(";
     print(T->getElementType(), OS, StringRef());
     OS << ")))) ";
     printBefore(T->getElementType(), OS);
     break;
   }
   case VectorType::SveFixedLengthDataVector:
   case VectorType::SveFixedLengthPredicateVector:
     // FIXME: We prefer to print the size directly here, but have no way
     // to get the size of the type.
     OS << "__attribute__((__arm_sve_vector_bits__(";
 
     if (T->getVectorKind() == VectorType::SveFixedLengthPredicateVector)
       // Predicates take a bit per byte of the vector size, multiply by 8 to
       // get the number of bits passed to the attribute.
       OS << T->getNumElements() * 8;
     else
       OS << T->getNumElements();
 
     OS << " * sizeof(";
     print(T->getElementType(), OS, StringRef());
     // Multiply by 8 for the number of bits.
     OS << ") * 8))) ";
     printBefore(T->getElementType(), OS);
   }
 }
 
 void TypePrinter::printVectorAfter(const VectorType *T, raw_ostream &OS) {
   printAfter(T->getElementType(), OS);
 }
 
 void TypePrinter::printDependentVectorBefore(
     const DependentVectorType *T, raw_ostream &OS) {
   switch (T->getVectorKind()) {
   case VectorType::AltiVecPixel:
     OS << "__vector __pixel ";
     break;
   case VectorType::AltiVecBool:
     OS << "__vector __bool ";
     printBefore(T->getElementType(), OS);
     break;
   case VectorType::AltiVecVector:
     OS << "__vector ";
     printBefore(T->getElementType(), OS);
     break;
   case VectorType::NeonVector:
     OS << "__attribute__((neon_vector_type(";
     if (T->getSizeExpr())
       T->getSizeExpr()->printPretty(OS, nullptr, Policy);
     OS << "))) ";
     printBefore(T->getElementType(), OS);
     break;
   case VectorType::NeonPolyVector:
     OS << "__attribute__((neon_polyvector_type(";
     if (T->getSizeExpr())
       T->getSizeExpr()->printPretty(OS, nullptr, Policy);
     OS << "))) ";
     printBefore(T->getElementType(), OS);
     break;
   case VectorType::GenericVector: {
     // FIXME: We prefer to print the size directly here, but have no way
     // to get the size of the type.
     OS << "__attribute__((__vector_size__(";
     if (T->getSizeExpr())
       T->getSizeExpr()->printPretty(OS, nullptr, Policy);
     OS << " * sizeof(";
     print(T->getElementType(), OS, StringRef());
     OS << ")))) ";
     printBefore(T->getElementType(), OS);
     break;
   }
   case VectorType::SveFixedLengthDataVector:
   case VectorType::SveFixedLengthPredicateVector:
     // FIXME: We prefer to print the size directly here, but have no way
     // to get the size of the type.
     OS << "__attribute__((__arm_sve_vector_bits__(";
     if (T->getSizeExpr()) {
       T->getSizeExpr()->printPretty(OS, nullptr, Policy);
       if (T->getVectorKind() == VectorType::SveFixedLengthPredicateVector)
         // Predicates take a bit per byte of the vector size, multiply by 8 to
         // get the number of bits passed to the attribute.
         OS << " * 8";
       OS << " * sizeof(";
       print(T->getElementType(), OS, StringRef());
       // Multiply by 8 for the number of bits.
       OS << ") * 8";
     }
     OS << "))) ";
     printBefore(T->getElementType(), OS);
   }
 }
 
 void TypePrinter::printDependentVectorAfter(
     const DependentVectorType *T, raw_ostream &OS) {
   printAfter(T->getElementType(), OS);
 }
 
 void TypePrinter::printExtVectorBefore(const ExtVectorType *T,
                                        raw_ostream &OS) {
   printBefore(T->getElementType(), OS);
 }
 
 void TypePrinter::printExtVectorAfter(const ExtVectorType *T, raw_ostream &OS) {
   printAfter(T->getElementType(), OS);
   OS << " __attribute__((ext_vector_type(";
   OS << T->getNumElements();
   OS << ")))";
 }
 
 void TypePrinter::printConstantMatrixBefore(const ConstantMatrixType *T,
                                             raw_ostream &OS) {
   printBefore(T->getElementType(), OS);
   OS << " __attribute__((matrix_type(";
   OS << T->getNumRows() << ", " << T->getNumColumns();
   OS << ")))";
 }
 
 void TypePrinter::printConstantMatrixAfter(const ConstantMatrixType *T,
                                            raw_ostream &OS) {
   printAfter(T->getElementType(), OS);
 }
 
 void TypePrinter::printDependentSizedMatrixBefore(
     const DependentSizedMatrixType *T, raw_ostream &OS) {
   printBefore(T->getElementType(), OS);
   OS << " __attribute__((matrix_type(";
   if (T->getRowExpr()) {
     T->getRowExpr()->printPretty(OS, nullptr, Policy);
   }
   OS << ", ";
   if (T->getColumnExpr()) {
     T->getColumnExpr()->printPretty(OS, nullptr, Policy);
   }
   OS << ")))";
 }
 
 void TypePrinter::printDependentSizedMatrixAfter(
     const DependentSizedMatrixType *T, raw_ostream &OS) {
   printAfter(T->getElementType(), OS);
 }
 
 void
 FunctionProtoType::printExceptionSpecification(raw_ostream &OS,
                                                const PrintingPolicy &Policy)
                                                                          const {
   if (hasDynamicExceptionSpec()) {
     OS << " throw(";
     if (getExceptionSpecType() == EST_MSAny)
       OS << "...";
     else
       for (unsigned I = 0, N = getNumExceptions(); I != N; ++I) {
         if (I)
           OS << ", ";
 
         OS << getExceptionType(I).stream(Policy);
       }
     OS << ')';
   } else if (EST_NoThrow == getExceptionSpecType()) {
     OS << " __attribute__((nothrow))";
   } else if (isNoexceptExceptionSpec(getExceptionSpecType())) {
     OS << " noexcept";
     // FIXME:Is it useful to print out the expression for a non-dependent
     // noexcept specification?
     if (isComputedNoexcept(getExceptionSpecType())) {
       OS << '(';
       if (getNoexceptExpr())
         getNoexceptExpr()->printPretty(OS, nullptr, Policy);
       OS << ')';
     }
   }
 }
 
 void TypePrinter::printFunctionProtoBefore(const FunctionProtoType *T,
                                            raw_ostream &OS) {
   if (T->hasTrailingReturn()) {
     OS << "auto ";
     if (!HasEmptyPlaceHolder)
       OS << '(';
   } else {
     // If needed for precedence reasons, wrap the inner part in grouping parens.
     SaveAndRestore<bool> PrevPHIsEmpty(HasEmptyPlaceHolder, false);
     printBefore(T->getReturnType(), OS);
     if (!PrevPHIsEmpty.get())
       OS << '(';
   }
 }
 
 StringRef clang::getParameterABISpelling(ParameterABI ABI) {
   switch (ABI) {
   case ParameterABI::Ordinary:
     llvm_unreachable("asking for spelling of ordinary parameter ABI");
   case ParameterABI::SwiftContext:
     return "swift_context";
   case ParameterABI::SwiftAsyncContext:
     return "swift_async_context";
   case ParameterABI::SwiftErrorResult:
     return "swift_error_result";
   case ParameterABI::SwiftIndirectResult:
     return "swift_indirect_result";
   }
   llvm_unreachable("bad parameter ABI kind");
 }
 
 void TypePrinter::printFunctionProtoAfter(const FunctionProtoType *T,
                                           raw_ostream &OS) {
   // If needed for precedence reasons, wrap the inner part in grouping parens.
   if (!HasEmptyPlaceHolder)
     OS << ')';
   SaveAndRestore<bool> NonEmptyPH(HasEmptyPlaceHolder, false);
 
   OS << '(';
   {
     ParamPolicyRAII ParamPolicy(Policy);
     for (unsigned i = 0, e = T->getNumParams(); i != e; ++i) {
       if (i) OS << ", ";
 
       auto EPI = T->getExtParameterInfo(i);
       if (EPI.isConsumed()) OS << "__attribute__((ns_consumed)) ";
       if (EPI.isNoEscape())
         OS << "__attribute__((noescape)) ";
       auto ABI = EPI.getABI();
       if (ABI != ParameterABI::Ordinary)
         OS << "__attribute__((" << getParameterABISpelling(ABI) << ")) ";
 
       print(T->getParamType(i), OS, StringRef());
     }
   }
 
   if (T->isVariadic()) {
     if (T->getNumParams())
       OS << ", ";
     OS << "...";
   } else if (T->getNumParams() == 0 && Policy.UseVoidForZeroParams) {
     // Do not emit int() if we have a proto, emit 'int(void)'.
     OS << "void";
   }
 
   OS << ')';
 
   FunctionType::ExtInfo Info = T->getExtInfo();
 
   printFunctionAfter(Info, OS);
 
   if (!T->getMethodQuals().empty())
     OS << " " << T->getMethodQuals().getAsString();
 
   switch (T->getRefQualifier()) {
   case RQ_None:
     break;
 
   case RQ_LValue:
     OS << " &";
     break;
 
   case RQ_RValue:
     OS << " &&";
     break;
   }
   T->printExceptionSpecification(OS, Policy);
 
   if (T->hasTrailingReturn()) {
     OS << " -> ";
     print(T->getReturnType(), OS, StringRef());
   } else
     printAfter(T->getReturnType(), OS);
 }
 
 void TypePrinter::printFunctionAfter(const FunctionType::ExtInfo &Info,
                                      raw_ostream &OS) {
   if (!InsideCCAttribute) {
     switch (Info.getCC()) {
     case CC_C:
       // The C calling convention is the default on the vast majority of platforms
       // we support.  If the user wrote it explicitly, it will usually be printed
       // while traversing the AttributedType.  If the type has been desugared, let
       // the canonical spelling be the implicit calling convention.
       // FIXME: It would be better to be explicit in certain contexts, such as a
       // cdecl function typedef used to declare a member function with the
       // Microsoft C++ ABI.
       break;
     case CC_X86StdCall:
       OS << " __attribute__((stdcall))";
       break;
     case CC_X86FastCall:
       OS << " __attribute__((fastcall))";
       break;
     case CC_X86ThisCall:
       OS << " __attribute__((thiscall))";
       break;
     case CC_X86VectorCall:
       OS << " __attribute__((vectorcall))";
       break;
     case CC_X86Pascal:
       OS << " __attribute__((pascal))";
       break;
     case CC_AAPCS:
       OS << " __attribute__((pcs(\"aapcs\")))";
       break;
     case CC_AAPCS_VFP:
       OS << " __attribute__((pcs(\"aapcs-vfp\")))";
       break;
     case CC_AArch64VectorCall:
       OS << "__attribute__((aarch64_vector_pcs))";
       break;
     case CC_IntelOclBicc:
       OS << " __attribute__((intel_ocl_bicc))";
       break;
     case CC_Win64:
       OS << " __attribute__((ms_abi))";
       break;
     case CC_X86_64SysV:
       OS << " __attribute__((sysv_abi))";
       break;
     case CC_X86RegCall:
       OS << " __attribute__((regcall))";
       break;
     case CC_SpirFunction:
     case CC_OpenCLKernel:
       // Do nothing. These CCs are not available as attributes.
       break;
     case CC_Swift:
       OS << " __attribute__((swiftcall))";
       break;
     case CC_SwiftAsync:
       OS << "__attribute__((swiftasynccall))";
       break;
     case CC_PreserveMost:
       OS << " __attribute__((preserve_most))";
       break;
     case CC_PreserveAll:
       OS << " __attribute__((preserve_all))";
       break;
     }
   }
 
   if (Info.getNoReturn())
     OS << " __attribute__((noreturn))";
   if (Info.getCmseNSCall())
     OS << " __attribute__((cmse_nonsecure_call))";
   if (Info.getProducesResult())
     OS << " __attribute__((ns_returns_retained))";
   if (Info.getRegParm())
     OS << " __attribute__((regparm ("
        << Info.getRegParm() << ")))";
   if (Info.getNoCallerSavedRegs())
     OS << " __attribute__((no_caller_saved_registers))";
   if (Info.getNoCfCheck())
     OS << " __attribute__((nocf_check))";
 }
 
 void TypePrinter::printFunctionNoProtoBefore(const FunctionNoProtoType *T,
                                              raw_ostream &OS) {
   // If needed for precedence reasons, wrap the inner part in grouping parens.
   SaveAndRestore<bool> PrevPHIsEmpty(HasEmptyPlaceHolder, false);
   printBefore(T->getReturnType(), OS);
   if (!PrevPHIsEmpty.get())
     OS << '(';
 }
 
 void TypePrinter::printFunctionNoProtoAfter(const FunctionNoProtoType *T,
                                             raw_ostream &OS) {
   // If needed for precedence reasons, wrap the inner part in grouping parens.
   if (!HasEmptyPlaceHolder)
     OS << ')';
   SaveAndRestore<bool> NonEmptyPH(HasEmptyPlaceHolder, false);
 
   OS << "()";
   printFunctionAfter(T->getExtInfo(), OS);
   printAfter(T->getReturnType(), OS);
 }
 
 void TypePrinter::printTypeSpec(NamedDecl *D, raw_ostream &OS) {
 
   // Compute the full nested-name-specifier for this type.
   // In C, this will always be empty except when the type
   // being printed is anonymous within other Record.
   if (!Policy.SuppressScope)
     AppendScope(D->getDeclContext(), OS, D->getDeclName());
 
   IdentifierInfo *II = D->getIdentifier();
   OS << II->getName();
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printUnresolvedUsingBefore(const UnresolvedUsingType *T,
                                              raw_ostream &OS) {
   printTypeSpec(T->getDecl(), OS);
 }
 
 void TypePrinter::printUnresolvedUsingAfter(const UnresolvedUsingType *T,
                                             raw_ostream &OS) {}
 
 void TypePrinter::printUsingBefore(const UsingType *T, raw_ostream &OS) {
   // After `namespace b { using a::X }`, is the type X within B a::X or b::X?
   //
   // - b::X is more formally correct given the UsingType model
   // - b::X makes sense if "re-exporting" a symbol in a new namespace
   // - a::X makes sense if "importing" a symbol for convenience
   //
   // The "importing" use seems much more common, so we print a::X.
   // This could be a policy option, but the right choice seems to rest more
   // with the intent of the code than the caller.
   printTypeSpec(T->getFoundDecl()->getUnderlyingDecl(), OS);
 }
 
 void TypePrinter::printUsingAfter(const UsingType *T, raw_ostream &OS) {}
 
 void TypePrinter::printTypedefBefore(const TypedefType *T, raw_ostream &OS) {
   printTypeSpec(T->getDecl(), OS);
 }
 
 void TypePrinter::printMacroQualifiedBefore(const MacroQualifiedType *T,
                                             raw_ostream &OS) {
   StringRef MacroName = T->getMacroIdentifier()->getName();
   OS << MacroName << " ";
 
   // Since this type is meant to print the macro instead of the whole attribute,
   // we trim any attributes and go directly to the original modified type.
   printBefore(T->getModifiedType(), OS);
 }
 
 void TypePrinter::printMacroQualifiedAfter(const MacroQualifiedType *T,
                                            raw_ostream &OS) {
   printAfter(T->getModifiedType(), OS);
 }
 
 void TypePrinter::printTypedefAfter(const TypedefType *T, raw_ostream &OS) {}
 
 void TypePrinter::printTypeOfExprBefore(const TypeOfExprType *T,
                                         raw_ostream &OS) {
   OS << "typeof ";
   if (T->getUnderlyingExpr())
     T->getUnderlyingExpr()->printPretty(OS, nullptr, Policy);
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printTypeOfExprAfter(const TypeOfExprType *T,
                                        raw_ostream &OS) {}
 
 void TypePrinter::printTypeOfBefore(const TypeOfType *T, raw_ostream &OS) {
   OS << "typeof(";
   print(T->getUnderlyingType(), OS, StringRef());
   OS << ')';
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printTypeOfAfter(const TypeOfType *T, raw_ostream &OS) {}
 
 void TypePrinter::printDecltypeBefore(const DecltypeType *T, raw_ostream &OS) {
   OS << "decltype(";
   if (T->getUnderlyingExpr())
     T->getUnderlyingExpr()->printPretty(OS, nullptr, Policy);
   OS << ')';
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printDecltypeAfter(const DecltypeType *T, raw_ostream &OS) {}
 
 void TypePrinter::printUnaryTransformBefore(const UnaryTransformType *T,
                                             raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
 
   switch (T->getUTTKind()) {
     case UnaryTransformType::EnumUnderlyingType:
       OS << "__underlying_type(";
       print(T->getBaseType(), OS, StringRef());
       OS << ')';
       spaceBeforePlaceHolder(OS);
       return;
   }
 
   printBefore(T->getBaseType(), OS);
 }
 
 void TypePrinter::printUnaryTransformAfter(const UnaryTransformType *T,
                                            raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
 
   switch (T->getUTTKind()) {
     case UnaryTransformType::EnumUnderlyingType:
       return;
   }
 
   printAfter(T->getBaseType(), OS);
 }
 
 void TypePrinter::printAutoBefore(const AutoType *T, raw_ostream &OS) {
   // If the type has been deduced, do not print 'auto'.
   if (!T->getDeducedType().isNull()) {
     printBefore(T->getDeducedType(), OS);
   } else {
     if (T->isConstrained()) {
       // FIXME: Track a TypeConstraint as type sugar, so that we can print the
       // type as it was written.
       T->getTypeConstraintConcept()->getDeclName().print(OS, Policy);
       auto Args = T->getTypeConstraintArguments();
       if (!Args.empty())
         printTemplateArgumentList(
             OS, Args, Policy,
             T->getTypeConstraintConcept()->getTemplateParameters());
       OS << ' ';
     }
     switch (T->getKeyword()) {
     case AutoTypeKeyword::Auto: OS << "auto"; break;
     case AutoTypeKeyword::DecltypeAuto: OS << "decltype(auto)"; break;
     case AutoTypeKeyword::GNUAutoType: OS << "__auto_type"; break;
     }
     spaceBeforePlaceHolder(OS);
   }
 }
 
 void TypePrinter::printAutoAfter(const AutoType *T, raw_ostream &OS) {
   // If the type has been deduced, do not print 'auto'.
   if (!T->getDeducedType().isNull())
     printAfter(T->getDeducedType(), OS);
 }
 
 void TypePrinter::printDeducedTemplateSpecializationBefore(
     const DeducedTemplateSpecializationType *T, raw_ostream &OS) {
   // If the type has been deduced, print the deduced type.
   if (!T->getDeducedType().isNull()) {
     printBefore(T->getDeducedType(), OS);
   } else {
     IncludeStrongLifetimeRAII Strong(Policy);
     T->getTemplateName().print(OS, Policy);
     spaceBeforePlaceHolder(OS);
   }
 }
 
 void TypePrinter::printDeducedTemplateSpecializationAfter(
     const DeducedTemplateSpecializationType *T, raw_ostream &OS) {
   // If the type has been deduced, print the deduced type.
   if (!T->getDeducedType().isNull())
     printAfter(T->getDeducedType(), OS);
 }
 
 void TypePrinter::printAtomicBefore(const AtomicType *T, raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
 
   OS << "_Atomic(";
   print(T->getValueType(), OS, StringRef());
   OS << ')';
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printAtomicAfter(const AtomicType *T, raw_ostream &OS) {}
 
 void TypePrinter::printPipeBefore(const PipeType *T, raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
 
   if (T->isReadOnly())
     OS << "read_only ";
   else
     OS << "write_only ";
   OS << "pipe ";
   print(T->getElementType(), OS, StringRef());
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printPipeAfter(const PipeType *T, raw_ostream &OS) {}
 
 void TypePrinter::printBitIntBefore(const BitIntType *T, raw_ostream &OS) {
   if (T->isUnsigned())
     OS << "unsigned ";
   OS << "_BitInt(" << T->getNumBits() << ")";
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printBitIntAfter(const BitIntType *T, raw_ostream &OS) {}
 
 void TypePrinter::printDependentBitIntBefore(const DependentBitIntType *T,
                                              raw_ostream &OS) {
   if (T->isUnsigned())
     OS << "unsigned ";
   OS << "_BitInt(";
   T->getNumBitsExpr()->printPretty(OS, nullptr, Policy);
   OS << ")";
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printDependentBitIntAfter(const DependentBitIntType *T,
                                             raw_ostream &OS) {}
 
 /// Appends the given scope to the end of a string.
 void TypePrinter::AppendScope(DeclContext *DC, raw_ostream &OS,
                               DeclarationName NameInScope) {
   if (DC->isTranslationUnit())
     return;
 
   // FIXME: Consider replacing this with NamedDecl::printNestedNameSpecifier,
   // which can also print names for function and method scopes.
   if (DC->isFunctionOrMethod())
     return;
 
   if (Policy.Callbacks && Policy.Callbacks->isScopeVisible(DC))
     return;
 
   if (const auto *NS = dyn_cast<NamespaceDecl>(DC)) {
     if (Policy.SuppressUnwrittenScope && NS->isAnonymousNamespace())
       return AppendScope(DC->getParent(), OS, NameInScope);
 
     // Only suppress an inline namespace if the name has the same lookup
     // results in the enclosing namespace.
     if (Policy.SuppressInlineNamespace && NS->isInline() && NameInScope &&
         NS->isRedundantInlineQualifierFor(NameInScope))
       return AppendScope(DC->getParent(), OS, NameInScope);
 
     AppendScope(DC->getParent(), OS, NS->getDeclName());
     if (NS->getIdentifier())
       OS << NS->getName() << "::";
     else
       OS << "(anonymous namespace)::";
   } else if (const auto *Spec = dyn_cast<ClassTemplateSpecializationDecl>(DC)) {
     AppendScope(DC->getParent(), OS, Spec->getDeclName());
     IncludeStrongLifetimeRAII Strong(Policy);
     OS << Spec->getIdentifier()->getName();
     const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
     printTemplateArgumentList(
         OS, TemplateArgs.asArray(), Policy,
         Spec->getSpecializedTemplate()->getTemplateParameters());
     OS << "::";
   } else if (const auto *Tag = dyn_cast<TagDecl>(DC)) {
     AppendScope(DC->getParent(), OS, Tag->getDeclName());
     if (TypedefNameDecl *Typedef = Tag->getTypedefNameForAnonDecl())
       OS << Typedef->getIdentifier()->getName() << "::";
     else if (Tag->getIdentifier())
       OS << Tag->getIdentifier()->getName() << "::";
     else
       return;
   } else {
     AppendScope(DC->getParent(), OS, NameInScope);
   }
 }
 
 void TypePrinter::printTag(TagDecl *D, raw_ostream &OS) {
   if (Policy.IncludeTagDefinition) {
     PrintingPolicy SubPolicy = Policy;
     SubPolicy.IncludeTagDefinition = false;
     D->print(OS, SubPolicy, Indentation);
     spaceBeforePlaceHolder(OS);
     return;
   }
 
   bool HasKindDecoration = false;
 
   // We don't print tags unless this is an elaborated type.
   // In C, we just assume every RecordType is an elaborated type.
   if (!Policy.SuppressTagKeyword && !D->getTypedefNameForAnonDecl()) {
     HasKindDecoration = true;
     OS << D->getKindName();
     OS << ' ';
   }
 
   // Compute the full nested-name-specifier for this type.
   // In C, this will always be empty except when the type
   // being printed is anonymous within other Record.
   if (!Policy.SuppressScope)
     AppendScope(D->getDeclContext(), OS, D->getDeclName());
 
   if (const IdentifierInfo *II = D->getIdentifier())
     OS << II->getName();
   else if (TypedefNameDecl *Typedef = D->getTypedefNameForAnonDecl()) {
     assert(Typedef->getIdentifier() && "Typedef without identifier?");
     OS << Typedef->getIdentifier()->getName();
   } else {
     // Make an unambiguous representation for anonymous types, e.g.
     //   (anonymous enum at /usr/include/string.h:120:9)
     OS << (Policy.MSVCFormatting ? '`' : '(');
 
     if (isa<CXXRecordDecl>(D) && cast<CXXRecordDecl>(D)->isLambda()) {
       OS << "lambda";
       HasKindDecoration = true;
     } else if ((isa<RecordDecl>(D) && cast<RecordDecl>(D)->isAnonymousStructOrUnion())) {
       OS << "anonymous";
     } else {
       OS << "unnamed";
     }
 
     if (Policy.AnonymousTagLocations) {
       // Suppress the redundant tag keyword if we just printed one.
       // We don't have to worry about ElaboratedTypes here because you can't
       // refer to an anonymous type with one.
       if (!HasKindDecoration)
         OS << " " << D->getKindName();
 
       PresumedLoc PLoc = D->getASTContext().getSourceManager().getPresumedLoc(
           D->getLocation());
       if (PLoc.isValid()) {
         OS << " at ";
         StringRef File = PLoc.getFilename();
         if (auto *Callbacks = Policy.Callbacks)
           OS << Callbacks->remapPath(File);
         else
           OS << File;
         OS << ':' << PLoc.getLine() << ':' << PLoc.getColumn();
       }
     }
 
     OS << (Policy.MSVCFormatting ? '\'' : ')');
   }
 
   // If this is a class template specialization, print the template
   // arguments.
   if (const auto *Spec = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
     ArrayRef<TemplateArgument> Args;
     TypeSourceInfo *TAW = Spec->getTypeAsWritten();
     if (!Policy.PrintCanonicalTypes && TAW) {
       const TemplateSpecializationType *TST =
         cast<TemplateSpecializationType>(TAW->getType());
       Args = TST->template_arguments();
     } else {
       const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
       Args = TemplateArgs.asArray();
     }
     IncludeStrongLifetimeRAII Strong(Policy);
     printTemplateArgumentList(
         OS, Args, Policy,
         Spec->getSpecializedTemplate()->getTemplateParameters());
   }
 
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printRecordBefore(const RecordType *T, raw_ostream &OS) {
   // Print the preferred name if we have one for this type.
   if (Policy.UsePreferredNames) {
     for (const auto *PNA : T->getDecl()->specific_attrs<PreferredNameAttr>()) {
       if (!declaresSameEntity(PNA->getTypedefType()->getAsCXXRecordDecl(),
                               T->getDecl()))
         continue;
       // Find the outermost typedef or alias template.
       QualType T = PNA->getTypedefType();
       while (true) {
         if (auto *TT = dyn_cast<TypedefType>(T))
           return printTypeSpec(TT->getDecl(), OS);
         if (auto *TST = dyn_cast<TemplateSpecializationType>(T))
           return printTemplateId(TST, OS, /*FullyQualify=*/true);
         T = T->getLocallyUnqualifiedSingleStepDesugaredType();
       }
     }
   }
 
   printTag(T->getDecl(), OS);
 }
 
 void TypePrinter::printRecordAfter(const RecordType *T, raw_ostream &OS) {}
 
 void TypePrinter::printEnumBefore(const EnumType *T, raw_ostream &OS) {
   printTag(T->getDecl(), OS);
 }
 
 void TypePrinter::printEnumAfter(const EnumType *T, raw_ostream &OS) {}
 
 void TypePrinter::printTemplateTypeParmBefore(const TemplateTypeParmType *T,
                                               raw_ostream &OS) {
   TemplateTypeParmDecl *D = T->getDecl();
   if (D && D->isImplicit()) {
     if (auto *TC = D->getTypeConstraint()) {
       TC->print(OS, Policy);
       OS << ' ';
     }
     OS << "auto";
   } else if (IdentifierInfo *Id = T->getIdentifier())
     OS << (Policy.CleanUglifiedParameters ? Id->deuglifiedName()
                                           : Id->getName());
   else
     OS << "type-parameter-" << T->getDepth() << '-' << T->getIndex();
 
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printTemplateTypeParmAfter(const TemplateTypeParmType *T,
                                              raw_ostream &OS) {}
 
 void TypePrinter::printSubstTemplateTypeParmBefore(
                                              const SubstTemplateTypeParmType *T,
                                              raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   printBefore(T->getReplacementType(), OS);
 }
 
 void TypePrinter::printSubstTemplateTypeParmAfter(
                                              const SubstTemplateTypeParmType *T,
                                              raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   printAfter(T->getReplacementType(), OS);
 }
 
 void TypePrinter::printSubstTemplateTypeParmPackBefore(
                                         const SubstTemplateTypeParmPackType *T,
                                         raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   printTemplateTypeParmBefore(T->getReplacedParameter(), OS);
 }
 
 void TypePrinter::printSubstTemplateTypeParmPackAfter(
                                         const SubstTemplateTypeParmPackType *T,
                                         raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
   printTemplateTypeParmAfter(T->getReplacedParameter(), OS);
 }
 
 void TypePrinter::printTemplateId(const TemplateSpecializationType *T,
                                   raw_ostream &OS, bool FullyQualify) {
   IncludeStrongLifetimeRAII Strong(Policy);
 
   TemplateDecl *TD = T->getTemplateName().getAsTemplateDecl();
   if (FullyQualify && TD) {
     if (!Policy.SuppressScope)
       AppendScope(TD->getDeclContext(), OS, TD->getDeclName());
 
-    IdentifierInfo *II = TD->getIdentifier();
-    OS << II->getName();
+    OS << TD->getName();
   } else {
     T->getTemplateName().print(OS, Policy);
   }
 
   printTemplateArgumentList(OS, T->template_arguments(), Policy);
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printTemplateSpecializationBefore(
                                             const TemplateSpecializationType *T,
                                             raw_ostream &OS) {
   printTemplateId(T, OS, Policy.FullyQualifiedName);
 }
 
 void TypePrinter::printTemplateSpecializationAfter(
                                             const TemplateSpecializationType *T,
                                             raw_ostream &OS) {}
 
 void TypePrinter::printInjectedClassNameBefore(const InjectedClassNameType *T,
                                                raw_ostream &OS) {
   if (Policy.PrintInjectedClassNameWithArguments)
     return printTemplateSpecializationBefore(T->getInjectedTST(), OS);
 
   IncludeStrongLifetimeRAII Strong(Policy);
   T->getTemplateName().print(OS, Policy);
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printInjectedClassNameAfter(const InjectedClassNameType *T,
                                                raw_ostream &OS) {}
 
 void TypePrinter::printElaboratedBefore(const ElaboratedType *T,
                                         raw_ostream &OS) {
   if (Policy.IncludeTagDefinition && T->getOwnedTagDecl()) {
     TagDecl *OwnedTagDecl = T->getOwnedTagDecl();
     assert(OwnedTagDecl->getTypeForDecl() == T->getNamedType().getTypePtr() &&
            "OwnedTagDecl expected to be a declaration for the type");
     PrintingPolicy SubPolicy = Policy;
     SubPolicy.IncludeTagDefinition = false;
     OwnedTagDecl->print(OS, SubPolicy, Indentation);
     spaceBeforePlaceHolder(OS);
     return;
   }
 
   // The tag definition will take care of these.
   if (!Policy.IncludeTagDefinition)
   {
     OS << TypeWithKeyword::getKeywordName(T->getKeyword());
     if (T->getKeyword() != ETK_None)
       OS << " ";
     NestedNameSpecifier *Qualifier = T->getQualifier();
     if (Qualifier)
       Qualifier->print(OS, Policy);
   }
 
   ElaboratedTypePolicyRAII PolicyRAII(Policy);
   printBefore(T->getNamedType(), OS);
 }
 
 void TypePrinter::printElaboratedAfter(const ElaboratedType *T,
                                         raw_ostream &OS) {
   if (Policy.IncludeTagDefinition && T->getOwnedTagDecl())
     return;
   ElaboratedTypePolicyRAII PolicyRAII(Policy);
   printAfter(T->getNamedType(), OS);
 }
 
 void TypePrinter::printParenBefore(const ParenType *T, raw_ostream &OS) {
   if (!HasEmptyPlaceHolder && !isa<FunctionType>(T->getInnerType())) {
     printBefore(T->getInnerType(), OS);
     OS << '(';
   } else
     printBefore(T->getInnerType(), OS);
 }
 
 void TypePrinter::printParenAfter(const ParenType *T, raw_ostream &OS) {
   if (!HasEmptyPlaceHolder && !isa<FunctionType>(T->getInnerType())) {
     OS << ')';
     printAfter(T->getInnerType(), OS);
   } else
     printAfter(T->getInnerType(), OS);
 }
 
 void TypePrinter::printDependentNameBefore(const DependentNameType *T,
                                            raw_ostream &OS) {
   OS << TypeWithKeyword::getKeywordName(T->getKeyword());
   if (T->getKeyword() != ETK_None)
     OS << " ";
 
   T->getQualifier()->print(OS, Policy);
 
   OS << T->getIdentifier()->getName();
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printDependentNameAfter(const DependentNameType *T,
                                           raw_ostream &OS) {}
 
 void TypePrinter::printDependentTemplateSpecializationBefore(
         const DependentTemplateSpecializationType *T, raw_ostream &OS) {
   IncludeStrongLifetimeRAII Strong(Policy);
 
   OS << TypeWithKeyword::getKeywordName(T->getKeyword());
   if (T->getKeyword() != ETK_None)
     OS << " ";
 
   if (T->getQualifier())
     T->getQualifier()->print(OS, Policy);
   OS << "template " << T->getIdentifier()->getName();
   printTemplateArgumentList(OS, T->template_arguments(), Policy);
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printDependentTemplateSpecializationAfter(
         const DependentTemplateSpecializationType *T, raw_ostream &OS) {}
 
 void TypePrinter::printPackExpansionBefore(const PackExpansionType *T,
                                            raw_ostream &OS) {
   printBefore(T->getPattern(), OS);
 }
 
 void TypePrinter::printPackExpansionAfter(const PackExpansionType *T,
                                           raw_ostream &OS) {
   printAfter(T->getPattern(), OS);
   OS << "...";
 }
 
 void TypePrinter::printAttributedBefore(const AttributedType *T,
                                         raw_ostream &OS) {
   // FIXME: Generate this with TableGen.
 
   // Prefer the macro forms of the GC and ownership qualifiers.
   if (T->getAttrKind() == attr::ObjCGC ||
       T->getAttrKind() == attr::ObjCOwnership)
     return printBefore(T->getEquivalentType(), OS);
 
   if (T->getAttrKind() == attr::ObjCKindOf)
     OS << "__kindof ";
 
   if (T->getAttrKind() == attr::AddressSpace)
     printBefore(T->getEquivalentType(), OS);
   else
     printBefore(T->getModifiedType(), OS);
 
   if (T->isMSTypeSpec()) {
     switch (T->getAttrKind()) {
     default: return;
     case attr::Ptr32: OS << " __ptr32"; break;
     case attr::Ptr64: OS << " __ptr64"; break;
     case attr::SPtr: OS << " __sptr"; break;
     case attr::UPtr: OS << " __uptr"; break;
     }
     spaceBeforePlaceHolder(OS);
   }
 
   // Print nullability type specifiers.
   if (T->getImmediateNullability()) {
     if (T->getAttrKind() == attr::TypeNonNull)
       OS << " _Nonnull";
     else if (T->getAttrKind() == attr::TypeNullable)
       OS << " _Nullable";
     else if (T->getAttrKind() == attr::TypeNullUnspecified)
       OS << " _Null_unspecified";
     else if (T->getAttrKind() == attr::TypeNullableResult)
       OS << " _Nullable_result";
     else
       llvm_unreachable("unhandled nullability");
     spaceBeforePlaceHolder(OS);
   }
 }
 
 void TypePrinter::printAttributedAfter(const AttributedType *T,
                                        raw_ostream &OS) {
   // FIXME: Generate this with TableGen.
 
   // Prefer the macro forms of the GC and ownership qualifiers.
   if (T->getAttrKind() == attr::ObjCGC ||
       T->getAttrKind() == attr::ObjCOwnership)
     return printAfter(T->getEquivalentType(), OS);
 
   // If this is a calling convention attribute, don't print the implicit CC from
   // the modified type.
   SaveAndRestore<bool> MaybeSuppressCC(InsideCCAttribute, T->isCallingConv());
 
   printAfter(T->getModifiedType(), OS);
 
   // Some attributes are printed as qualifiers before the type, so we have
   // nothing left to do.
   if (T->getAttrKind() == attr::ObjCKindOf ||
       T->isMSTypeSpec() || T->getImmediateNullability())
     return;
 
   // Don't print the inert __unsafe_unretained attribute at all.
   if (T->getAttrKind() == attr::ObjCInertUnsafeUnretained)
     return;
 
   // Don't print ns_returns_retained unless it had an effect.
   if (T->getAttrKind() == attr::NSReturnsRetained &&
       !T->getEquivalentType()->castAs<FunctionType>()
                              ->getExtInfo().getProducesResult())
     return;
 
   if (T->getAttrKind() == attr::LifetimeBound) {
     OS << " [[clang::lifetimebound]]";
     return;
   }
 
   // The printing of the address_space attribute is handled by the qualifier
   // since it is still stored in the qualifier. Return early to prevent printing
   // this twice.
   if (T->getAttrKind() == attr::AddressSpace)
     return;
 
   OS << " __attribute__((";
   switch (T->getAttrKind()) {
 #define TYPE_ATTR(NAME)
 #define DECL_OR_TYPE_ATTR(NAME)
 #define ATTR(NAME) case attr::NAME:
 #include "clang/Basic/AttrList.inc"
     llvm_unreachable("non-type attribute attached to type");
 
   case attr::OpenCLPrivateAddressSpace:
   case attr::OpenCLGlobalAddressSpace:
   case attr::OpenCLGlobalDeviceAddressSpace:
   case attr::OpenCLGlobalHostAddressSpace:
   case attr::OpenCLLocalAddressSpace:
   case attr::OpenCLConstantAddressSpace:
   case attr::OpenCLGenericAddressSpace:
     // FIXME: Update printAttributedBefore to print these once we generate
     // AttributedType nodes for them.
     break;
 
   case attr::LifetimeBound:
   case attr::TypeNonNull:
   case attr::TypeNullable:
   case attr::TypeNullableResult:
   case attr::TypeNullUnspecified:
   case attr::ObjCGC:
   case attr::ObjCInertUnsafeUnretained:
   case attr::ObjCKindOf:
   case attr::ObjCOwnership:
   case attr::Ptr32:
   case attr::Ptr64:
   case attr::SPtr:
   case attr::UPtr:
   case attr::AddressSpace:
   case attr::CmseNSCall:
     llvm_unreachable("This attribute should have been handled already");
 
   case attr::NSReturnsRetained:
     OS << "ns_returns_retained";
     break;
 
   // FIXME: When Sema learns to form this AttributedType, avoid printing the
   // attribute again in printFunctionProtoAfter.
   case attr::AnyX86NoCfCheck: OS << "nocf_check"; break;
   case attr::CDecl: OS << "cdecl"; break;
   case attr::FastCall: OS << "fastcall"; break;
   case attr::StdCall: OS << "stdcall"; break;
   case attr::ThisCall: OS << "thiscall"; break;
   case attr::SwiftCall: OS << "swiftcall"; break;
   case attr::SwiftAsyncCall: OS << "swiftasynccall"; break;
   case attr::VectorCall: OS << "vectorcall"; break;
   case attr::Pascal: OS << "pascal"; break;
   case attr::MSABI: OS << "ms_abi"; break;
   case attr::SysVABI: OS << "sysv_abi"; break;
   case attr::RegCall: OS << "regcall"; break;
   case attr::Pcs: {
     OS << "pcs(";
    QualType t = T->getEquivalentType();
    while (!t->isFunctionType())
      t = t->getPointeeType();
    OS << (t->castAs<FunctionType>()->getCallConv() == CC_AAPCS ?
          "\"aapcs\"" : "\"aapcs-vfp\"");
    OS << ')';
    break;
   }
   case attr::AArch64VectorPcs: OS << "aarch64_vector_pcs"; break;
   case attr::IntelOclBicc: OS << "inteloclbicc"; break;
   case attr::PreserveMost:
     OS << "preserve_most";
     break;
 
   case attr::PreserveAll:
     OS << "preserve_all";
     break;
   case attr::NoDeref:
     OS << "noderef";
     break;
   case attr::AcquireHandle:
     OS << "acquire_handle";
     break;
   case attr::ArmMveStrictPolymorphism:
     OS << "__clang_arm_mve_strict_polymorphism";
     break;
   case attr::BTFTypeTag:
     OS << "btf_type_tag";
     break;
   }
   OS << "))";
 }
 
 void TypePrinter::printObjCInterfaceBefore(const ObjCInterfaceType *T,
                                            raw_ostream &OS) {
   OS << T->getDecl()->getName();
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printObjCInterfaceAfter(const ObjCInterfaceType *T,
                                           raw_ostream &OS) {}
 
 void TypePrinter::printObjCTypeParamBefore(const ObjCTypeParamType *T,
                                           raw_ostream &OS) {
   OS << T->getDecl()->getName();
   if (!T->qual_empty()) {
     bool isFirst = true;
     OS << '<';
     for (const auto *I : T->quals()) {
       if (isFirst)
         isFirst = false;
       else
         OS << ',';
       OS << I->getName();
     }
     OS << '>';
   }
 
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printObjCTypeParamAfter(const ObjCTypeParamType *T,
                                           raw_ostream &OS) {}
 
 void TypePrinter::printObjCObjectBefore(const ObjCObjectType *T,
                                         raw_ostream &OS) {
   if (T->qual_empty() && T->isUnspecializedAsWritten() &&
       !T->isKindOfTypeAsWritten())
     return printBefore(T->getBaseType(), OS);
 
   if (T->isKindOfTypeAsWritten())
     OS << "__kindof ";
 
   print(T->getBaseType(), OS, StringRef());
 
   if (T->isSpecializedAsWritten()) {
     bool isFirst = true;
     OS << '<';
     for (auto typeArg : T->getTypeArgsAsWritten()) {
       if (isFirst)
         isFirst = false;
       else
         OS << ",";
 
       print(typeArg, OS, StringRef());
     }
     OS << '>';
   }
 
   if (!T->qual_empty()) {
     bool isFirst = true;
     OS << '<';
     for (const auto *I : T->quals()) {
       if (isFirst)
         isFirst = false;
       else
         OS << ',';
       OS << I->getName();
     }
     OS << '>';
   }
 
   spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printObjCObjectAfter(const ObjCObjectType *T,
                                         raw_ostream &OS) {
   if (T->qual_empty() && T->isUnspecializedAsWritten() &&
       !T->isKindOfTypeAsWritten())
     return printAfter(T->getBaseType(), OS);
 }
 
 void TypePrinter::printObjCObjectPointerBefore(const ObjCObjectPointerType *T,
                                                raw_ostream &OS) {
   printBefore(T->getPointeeType(), OS);
 
   // If we need to print the pointer, print it now.
   if (!T->isObjCIdType() && !T->isObjCQualifiedIdType() &&
       !T->isObjCClassType() && !T->isObjCQualifiedClassType()) {
     if (HasEmptyPlaceHolder)
       OS << ' ';
     OS << '*';
   }
 }
 
 void TypePrinter::printObjCObjectPointerAfter(const ObjCObjectPointerType *T,
                                               raw_ostream &OS) {}
 
 static
 const TemplateArgument &getArgument(const TemplateArgument &A) { return A; }
 
 static const TemplateArgument &getArgument(const TemplateArgumentLoc &A) {
   return A.getArgument();
 }
 
 static void printArgument(const TemplateArgument &A, const PrintingPolicy &PP,
                           llvm::raw_ostream &OS, bool IncludeType) {
   A.print(PP, OS, IncludeType);
 }
 
 static void printArgument(const TemplateArgumentLoc &A,
                           const PrintingPolicy &PP, llvm::raw_ostream &OS,
                           bool IncludeType) {
   const TemplateArgument::ArgKind &Kind = A.getArgument().getKind();
   if (Kind == TemplateArgument::ArgKind::Type)
     return A.getTypeSourceInfo()->getType().print(OS, PP);
   return A.getArgument().print(PP, OS, IncludeType);
 }
 
 static bool isSubstitutedTemplateArgument(ASTContext &Ctx, TemplateArgument Arg,
                                           TemplateArgument Pattern,
                                           ArrayRef<TemplateArgument> Args,
                                           unsigned Depth);
 
 static bool isSubstitutedType(ASTContext &Ctx, QualType T, QualType Pattern,
                               ArrayRef<TemplateArgument> Args, unsigned Depth) {
   if (Ctx.hasSameType(T, Pattern))
     return true;
 
   // A type parameter matches its argument.
   if (auto *TTPT = Pattern->getAs<TemplateTypeParmType>()) {
     if (TTPT->getDepth() == Depth && TTPT->getIndex() < Args.size() &&
         Args[TTPT->getIndex()].getKind() == TemplateArgument::Type) {
       QualType SubstArg = Ctx.getQualifiedType(
           Args[TTPT->getIndex()].getAsType(), Pattern.getQualifiers());
       return Ctx.hasSameType(SubstArg, T);
     }
     return false;
   }
 
   // FIXME: Recurse into array types.
 
   // All other cases will need the types to be identically qualified.
   Qualifiers TQual, PatQual;
   T = Ctx.getUnqualifiedArrayType(T, TQual);
   Pattern = Ctx.getUnqualifiedArrayType(Pattern, PatQual);
   if (TQual != PatQual)
     return false;
 
   // Recurse into pointer-like types.
   {
     QualType TPointee = T->getPointeeType();
     QualType PPointee = Pattern->getPointeeType();
     if (!TPointee.isNull() && !PPointee.isNull())
       return T->getTypeClass() == Pattern->getTypeClass() &&
              isSubstitutedType(Ctx, TPointee, PPointee, Args, Depth);
   }
 
   // Recurse into template specialization types.
   if (auto *PTST =
           Pattern.getCanonicalType()->getAs<TemplateSpecializationType>()) {
     TemplateName Template;
     ArrayRef<TemplateArgument> TemplateArgs;
     if (auto *TTST = T->getAs<TemplateSpecializationType>()) {
       Template = TTST->getTemplateName();
       TemplateArgs = TTST->template_arguments();
     } else if (auto *CTSD = dyn_cast_or_null<ClassTemplateSpecializationDecl>(
                    T->getAsCXXRecordDecl())) {
       Template = TemplateName(CTSD->getSpecializedTemplate());
       TemplateArgs = CTSD->getTemplateArgs().asArray();
     } else {
       return false;
     }
 
     if (!isSubstitutedTemplateArgument(Ctx, Template, PTST->getTemplateName(),
                                        Args, Depth))
       return false;
     if (TemplateArgs.size() != PTST->getNumArgs())
       return false;
     for (unsigned I = 0, N = TemplateArgs.size(); I != N; ++I)
       if (!isSubstitutedTemplateArgument(Ctx, TemplateArgs[I], PTST->getArg(I),
                                          Args, Depth))
         return false;
     return true;
   }
 
   // FIXME: Handle more cases.
   return false;
 }
 
 static bool isSubstitutedTemplateArgument(ASTContext &Ctx, TemplateArgument Arg,
                                           TemplateArgument Pattern,
                                           ArrayRef<TemplateArgument> Args,
                                           unsigned Depth) {
   Arg = Ctx.getCanonicalTemplateArgument(Arg);
   Pattern = Ctx.getCanonicalTemplateArgument(Pattern);
   if (Arg.structurallyEquals(Pattern))
     return true;
 
   if (Pattern.getKind() == TemplateArgument::Expression) {
     if (auto *DRE =
             dyn_cast<DeclRefExpr>(Pattern.getAsExpr()->IgnoreParenImpCasts())) {
       if (auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(DRE->getDecl()))
         return NTTP->getDepth() == Depth && Args.size() > NTTP->getIndex() &&
                Args[NTTP->getIndex()].structurallyEquals(Arg);
     }
   }
 
   if (Arg.getKind() != Pattern.getKind())
     return false;
 
   if (Arg.getKind() == TemplateArgument::Type)
     return isSubstitutedType(Ctx, Arg.getAsType(), Pattern.getAsType(), Args,
                              Depth);
 
   if (Arg.getKind() == TemplateArgument::Template) {
     TemplateDecl *PatTD = Pattern.getAsTemplate().getAsTemplateDecl();
     if (auto *TTPD = dyn_cast_or_null<TemplateTemplateParmDecl>(PatTD))
       return TTPD->getDepth() == Depth && Args.size() > TTPD->getIndex() &&
              Ctx.getCanonicalTemplateArgument(Args[TTPD->getIndex()])
                  .structurallyEquals(Arg);
   }
 
   // FIXME: Handle more cases.
   return false;
 }
 
 /// Make a best-effort determination of whether the type T can be produced by
 /// substituting Args into the default argument of Param.
 static bool isSubstitutedDefaultArgument(ASTContext &Ctx, TemplateArgument Arg,
                                          const NamedDecl *Param,
                                          ArrayRef<TemplateArgument> Args,
                                          unsigned Depth) {
   // An empty pack is equivalent to not providing a pack argument.
   if (Arg.getKind() == TemplateArgument::Pack && Arg.pack_size() == 0)
     return true;
 
   if (auto *TTPD = dyn_cast<TemplateTypeParmDecl>(Param)) {
     return TTPD->hasDefaultArgument() &&
            isSubstitutedTemplateArgument(Ctx, Arg, TTPD->getDefaultArgument(),
                                          Args, Depth);
   } else if (auto *TTPD = dyn_cast<TemplateTemplateParmDecl>(Param)) {
     return TTPD->hasDefaultArgument() &&
            isSubstitutedTemplateArgument(
                Ctx, Arg, TTPD->getDefaultArgument().getArgument(), Args, Depth);
   } else if (auto *NTTPD = dyn_cast<NonTypeTemplateParmDecl>(Param)) {
     return NTTPD->hasDefaultArgument() &&
            isSubstitutedTemplateArgument(Ctx, Arg, NTTPD->getDefaultArgument(),
                                          Args, Depth);
   }
   return false;
 }
 
 template <typename TA>
 static void
 printTo(raw_ostream &OS, ArrayRef<TA> Args, const PrintingPolicy &Policy,
         const TemplateParameterList *TPL, bool IsPack, unsigned ParmIndex) {
   // Drop trailing template arguments that match default arguments.
   if (TPL && Policy.SuppressDefaultTemplateArgs &&
       !Policy.PrintCanonicalTypes && !Args.empty() && !IsPack &&
       Args.size() <= TPL->size()) {
     ASTContext &Ctx = TPL->getParam(0)->getASTContext();
     llvm::SmallVector<TemplateArgument, 8> OrigArgs;
     for (const TA &A : Args)
       OrigArgs.push_back(getArgument(A));
     while (!Args.empty() &&
            isSubstitutedDefaultArgument(Ctx, getArgument(Args.back()),
                                         TPL->getParam(Args.size() - 1),
                                         OrigArgs, TPL->getDepth()))
       Args = Args.drop_back();
   }
 
   const char *Comma = Policy.MSVCFormatting ? "," : ", ";
   if (!IsPack)
     OS << '<';
 
   bool NeedSpace = false;
   bool FirstArg = true;
   for (const auto &Arg : Args) {
     // Print the argument into a string.
     SmallString<128> Buf;
     llvm::raw_svector_ostream ArgOS(Buf);
     const TemplateArgument &Argument = getArgument(Arg);
     if (Argument.getKind() == TemplateArgument::Pack) {
       if (Argument.pack_size() && !FirstArg)
         OS << Comma;
       printTo(ArgOS, Argument.getPackAsArray(), Policy, TPL,
               /*IsPack*/ true, ParmIndex);
     } else {
       if (!FirstArg)
         OS << Comma;
       // Tries to print the argument with location info if exists.
       printArgument(Arg, Policy, ArgOS,
                     TemplateParameterList::shouldIncludeTypeForArgument(
                         Policy, TPL, ParmIndex));
     }
     StringRef ArgString = ArgOS.str();
 
     // If this is the first argument and its string representation
     // begins with the global scope specifier ('::foo'), add a space
     // to avoid printing the diagraph '<:'.
     if (FirstArg && !ArgString.empty() && ArgString[0] == ':')
       OS << ' ';
 
     OS << ArgString;
 
     // If the last character of our string is '>', add another space to
     // keep the two '>''s separate tokens.
     if (!ArgString.empty()) {
       NeedSpace = Policy.SplitTemplateClosers && ArgString.back() == '>';
       FirstArg = false;
     }
 
     // Use same template parameter for all elements of Pack
     if (!IsPack)
       ParmIndex++;
   }
 
   if (!IsPack) {
     if (NeedSpace)
       OS << ' ';
     OS << '>';
   }
 }
 
 void clang::printTemplateArgumentList(raw_ostream &OS,
                                       const TemplateArgumentListInfo &Args,
                                       const PrintingPolicy &Policy,
                                       const TemplateParameterList *TPL) {
   printTemplateArgumentList(OS, Args.arguments(), Policy, TPL);
 }
 
 void clang::printTemplateArgumentList(raw_ostream &OS,
                                       ArrayRef<TemplateArgument> Args,
                                       const PrintingPolicy &Policy,
                                       const TemplateParameterList *TPL) {
   printTo(OS, Args, Policy, TPL, /*isPack*/ false, /*parmIndex*/ 0);
 }
 
 void clang::printTemplateArgumentList(raw_ostream &OS,
                                       ArrayRef<TemplateArgumentLoc> Args,
                                       const PrintingPolicy &Policy,
                                       const TemplateParameterList *TPL) {
   printTo(OS, Args, Policy, TPL, /*isPack*/ false, /*parmIndex*/ 0);
 }
 
 std::string Qualifiers::getAsString() const {
   LangOptions LO;
   return getAsString(PrintingPolicy(LO));
 }
 
 // Appends qualifiers to the given string, separated by spaces.  Will
 // prefix a space if the string is non-empty.  Will not append a final
 // space.
 std::string Qualifiers::getAsString(const PrintingPolicy &Policy) const {
   SmallString<64> Buf;
   llvm::raw_svector_ostream StrOS(Buf);
   print(StrOS, Policy);
   return std::string(StrOS.str());
 }
 
 bool Qualifiers::isEmptyWhenPrinted(const PrintingPolicy &Policy) const {
   if (getCVRQualifiers())
     return false;
 
   if (getAddressSpace() != LangAS::Default)
     return false;
 
   if (getObjCGCAttr())
     return false;
 
   if (Qualifiers::ObjCLifetime lifetime = getObjCLifetime())
     if (!(lifetime == Qualifiers::OCL_Strong && Policy.SuppressStrongLifetime))
       return false;
 
   return true;
 }
 
 std::string Qualifiers::getAddrSpaceAsString(LangAS AS) {
   switch (AS) {
   case LangAS::Default:
     return "";
   case LangAS::opencl_global:
   case LangAS::sycl_global:
     return "__global";
   case LangAS::opencl_local:
   case LangAS::sycl_local:
     return "__local";
   case LangAS::opencl_private:
   case LangAS::sycl_private:
     return "__private";
   case LangAS::opencl_constant:
     return "__constant";
   case LangAS::opencl_generic:
     return "__generic";
   case LangAS::opencl_global_device:
   case LangAS::sycl_global_device:
     return "__global_device";
   case LangAS::opencl_global_host:
   case LangAS::sycl_global_host:
     return "__global_host";
   case LangAS::cuda_device:
     return "__device__";
   case LangAS::cuda_constant:
     return "__constant__";
   case LangAS::cuda_shared:
     return "__shared__";
   case LangAS::ptr32_sptr:
     return "__sptr __ptr32";
   case LangAS::ptr32_uptr:
     return "__uptr __ptr32";
   case LangAS::ptr64:
     return "__ptr64";
   default:
     return std::to_string(toTargetAddressSpace(AS));
   }
 }
 
 // Appends qualifiers to the given string, separated by spaces.  Will
 // prefix a space if the string is non-empty.  Will not append a final
 // space.
 void Qualifiers::print(raw_ostream &OS, const PrintingPolicy& Policy,
                        bool appendSpaceIfNonEmpty) const {
   bool addSpace = false;
 
   unsigned quals = getCVRQualifiers();
   if (quals) {
     AppendTypeQualList(OS, quals, Policy.Restrict);
     addSpace = true;
   }
   if (hasUnaligned()) {
     if (addSpace)
       OS << ' ';
     OS << "__unaligned";
     addSpace = true;
   }
   auto ASStr = getAddrSpaceAsString(getAddressSpace());
   if (!ASStr.empty()) {
     if (addSpace)
       OS << ' ';
     addSpace = true;
     // Wrap target address space into an attribute syntax
     if (isTargetAddressSpace(getAddressSpace()))
       OS << "__attribute__((address_space(" << ASStr << ")))";
     else
       OS << ASStr;
   }
 
   if (Qualifiers::GC gc = getObjCGCAttr()) {
     if (addSpace)
       OS << ' ';
     addSpace = true;
     if (gc == Qualifiers::Weak)
       OS << "__weak";
     else
       OS << "__strong";
   }
   if (Qualifiers::ObjCLifetime lifetime = getObjCLifetime()) {
     if (!(lifetime == Qualifiers::OCL_Strong && Policy.SuppressStrongLifetime)){
       if (addSpace)
         OS << ' ';
       addSpace = true;
     }
 
     switch (lifetime) {
     case Qualifiers::OCL_None: llvm_unreachable("none but true");
     case Qualifiers::OCL_ExplicitNone: OS << "__unsafe_unretained"; break;
     case Qualifiers::OCL_Strong:
       if (!Policy.SuppressStrongLifetime)
         OS << "__strong";
       break;
 
     case Qualifiers::OCL_Weak: OS << "__weak"; break;
     case Qualifiers::OCL_Autoreleasing: OS << "__autoreleasing"; break;
     }
   }
 
   if (appendSpaceIfNonEmpty && addSpace)
     OS << ' ';
 }
 
 std::string QualType::getAsString() const {
   return getAsString(split(), LangOptions());
 }
 
 std::string QualType::getAsString(const PrintingPolicy &Policy) const {
   std::string S;
   getAsStringInternal(S, Policy);
   return S;
 }
 
 std::string QualType::getAsString(const Type *ty, Qualifiers qs,
                                   const PrintingPolicy &Policy) {
   std::string buffer;
   getAsStringInternal(ty, qs, buffer, Policy);
   return buffer;
 }
 
 void QualType::print(raw_ostream &OS, const PrintingPolicy &Policy,
                      const Twine &PlaceHolder, unsigned Indentation) const {
   print(splitAccordingToPolicy(*this, Policy), OS, Policy, PlaceHolder,
         Indentation);
 }
 
 void QualType::print(const Type *ty, Qualifiers qs,
                      raw_ostream &OS, const PrintingPolicy &policy,
                      const Twine &PlaceHolder, unsigned Indentation) {
   SmallString<128> PHBuf;
   StringRef PH = PlaceHolder.toStringRef(PHBuf);
 
   TypePrinter(policy, Indentation).print(ty, qs, OS, PH);
 }
 
 void QualType::getAsStringInternal(std::string &Str,
                                    const PrintingPolicy &Policy) const {
   return getAsStringInternal(splitAccordingToPolicy(*this, Policy), Str,
                              Policy);
 }
 
 void QualType::getAsStringInternal(const Type *ty, Qualifiers qs,
                                    std::string &buffer,
                                    const PrintingPolicy &policy) {
   SmallString<256> Buf;
   llvm::raw_svector_ostream StrOS(Buf);
   TypePrinter(policy).print(ty, qs, StrOS, buffer);
   std::string str = std::string(StrOS.str());
   buffer.swap(str);
 }
diff --git a/contrib/llvm-project/clang/lib/Basic/TargetInfo.cpp b/contrib/llvm-project/clang/lib/Basic/TargetInfo.cpp
index e3a2f30febe7..188ffb5f2f78 100644
--- a/contrib/llvm-project/clang/lib/Basic/TargetInfo.cpp
+++ b/contrib/llvm-project/clang/lib/Basic/TargetInfo.cpp
@@ -1,888 +1,888 @@
 //===--- TargetInfo.cpp - Information about Target machine ----------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //  This file implements the TargetInfo and TargetInfoImpl interfaces.
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/AddressSpaces.h"
 #include "clang/Basic/CharInfo.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/LangOptions.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/TargetParser.h"
 #include <cstdlib>
 using namespace clang;
 
 static const LangASMap DefaultAddrSpaceMap = {0};
 
 // TargetInfo Constructor.
 TargetInfo::TargetInfo(const llvm::Triple &T) : Triple(T) {
   // Set defaults.  Defaults are set for a 32-bit RISC platform, like PPC or
   // SPARC.  These should be overridden by concrete targets as needed.
   BigEndian = !T.isLittleEndian();
   TLSSupported = true;
   VLASupported = true;
   NoAsmVariants = false;
   HasLegalHalfType = false;
   HasFloat128 = false;
   HasIbm128 = false;
   HasFloat16 = false;
   HasBFloat16 = false;
   HasLongDouble = true;
   HasFPReturn = true;
   HasStrictFP = false;
   PointerWidth = PointerAlign = 32;
   BoolWidth = BoolAlign = 8;
   IntWidth = IntAlign = 32;
   LongWidth = LongAlign = 32;
   LongLongWidth = LongLongAlign = 64;
 
   // Fixed point default bit widths
   ShortAccumWidth = ShortAccumAlign = 16;
   AccumWidth = AccumAlign = 32;
   LongAccumWidth = LongAccumAlign = 64;
   ShortFractWidth = ShortFractAlign = 8;
   FractWidth = FractAlign = 16;
   LongFractWidth = LongFractAlign = 32;
 
   // Fixed point default integral and fractional bit sizes
   // We give the _Accum 1 fewer fractional bits than their corresponding _Fract
   // types by default to have the same number of fractional bits between _Accum
   // and _Fract types.
   PaddingOnUnsignedFixedPoint = false;
   ShortAccumScale = 7;
   AccumScale = 15;
   LongAccumScale = 31;
 
   SuitableAlign = 64;
   DefaultAlignForAttributeAligned = 128;
   MinGlobalAlign = 0;
   // From the glibc documentation, on GNU systems, malloc guarantees 16-byte
   // alignment on 64-bit systems and 8-byte alignment on 32-bit systems. See
   // https://www.gnu.org/software/libc/manual/html_node/Malloc-Examples.html.
-  // This alignment guarantee also applies to Windows and Android. On Darwin,
-  // the alignment is 16 bytes on both 64-bit and 32-bit systems.
+  // This alignment guarantee also applies to Windows and Android. On Darwin
+  // and OpenBSD, the alignment is 16 bytes on both 64-bit and 32-bit systems.
   if (T.isGNUEnvironment() || T.isWindowsMSVCEnvironment() || T.isAndroid())
     NewAlign = Triple.isArch64Bit() ? 128 : Triple.isArch32Bit() ? 64 : 0;
-  else if (T.isOSDarwin())
+  else if (T.isOSDarwin() || T.isOSOpenBSD())
     NewAlign = 128;
   else
     NewAlign = 0; // Infer from basic type alignment.
   HalfWidth = 16;
   HalfAlign = 16;
   FloatWidth = 32;
   FloatAlign = 32;
   DoubleWidth = 64;
   DoubleAlign = 64;
   LongDoubleWidth = 64;
   LongDoubleAlign = 64;
   Float128Align = 128;
   Ibm128Align = 128;
   LargeArrayMinWidth = 0;
   LargeArrayAlign = 0;
   MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 0;
   MaxVectorAlign = 0;
   MaxTLSAlign = 0;
   SimdDefaultAlign = 0;
   SizeType = UnsignedLong;
   PtrDiffType = SignedLong;
   IntMaxType = SignedLongLong;
   IntPtrType = SignedLong;
   WCharType = SignedInt;
   WIntType = SignedInt;
   Char16Type = UnsignedShort;
   Char32Type = UnsignedInt;
   Int64Type = SignedLongLong;
   Int16Type = SignedShort;
   SigAtomicType = SignedInt;
   ProcessIDType = SignedInt;
   UseSignedCharForObjCBool = true;
   UseBitFieldTypeAlignment = true;
   UseZeroLengthBitfieldAlignment = false;
   UseLeadingZeroLengthBitfield = true;
   UseExplicitBitFieldAlignment = true;
   ZeroLengthBitfieldBoundary = 0;
   MaxAlignedAttribute = 0;
   HalfFormat = &llvm::APFloat::IEEEhalf();
   FloatFormat = &llvm::APFloat::IEEEsingle();
   DoubleFormat = &llvm::APFloat::IEEEdouble();
   LongDoubleFormat = &llvm::APFloat::IEEEdouble();
   Float128Format = &llvm::APFloat::IEEEquad();
   Ibm128Format = &llvm::APFloat::PPCDoubleDouble();
   MCountName = "mcount";
   UserLabelPrefix = "_";
   RegParmMax = 0;
   SSERegParmMax = 0;
   HasAlignMac68kSupport = false;
   HasBuiltinMSVaList = false;
   IsRenderScriptTarget = false;
   HasAArch64SVETypes = false;
   HasRISCVVTypes = false;
   AllowAMDGPUUnsafeFPAtomics = false;
   ARMCDECoprocMask = 0;
 
   // Default to no types using fpret.
   RealTypeUsesObjCFPRet = 0;
 
   // Default to not using fp2ret for __Complex long double
   ComplexLongDoubleUsesFP2Ret = false;
 
   // Set the C++ ABI based on the triple.
   TheCXXABI.set(Triple.isKnownWindowsMSVCEnvironment()
                     ? TargetCXXABI::Microsoft
                     : TargetCXXABI::GenericItanium);
 
   // Default to an empty address space map.
   AddrSpaceMap = &DefaultAddrSpaceMap;
   UseAddrSpaceMapMangling = false;
 
   // Default to an unknown platform name.
   PlatformName = "unknown";
   PlatformMinVersion = VersionTuple();
 
   MaxOpenCLWorkGroupSize = 1024;
   ProgramAddrSpace = 0;
 }
 
 // Out of line virtual dtor for TargetInfo.
 TargetInfo::~TargetInfo() {}
 
 void TargetInfo::resetDataLayout(StringRef DL, const char *ULP) {
   DataLayoutString = DL.str();
   UserLabelPrefix = ULP;
 }
 
 bool
 TargetInfo::checkCFProtectionBranchSupported(DiagnosticsEngine &Diags) const {
   Diags.Report(diag::err_opt_not_valid_on_target) << "cf-protection=branch";
   return false;
 }
 
 bool
 TargetInfo::checkCFProtectionReturnSupported(DiagnosticsEngine &Diags) const {
   Diags.Report(diag::err_opt_not_valid_on_target) << "cf-protection=return";
   return false;
 }
 
 /// getTypeName - Return the user string for the specified integer type enum.
 /// For example, SignedShort -> "short".
 const char *TargetInfo::getTypeName(IntType T) {
   switch (T) {
   default: llvm_unreachable("not an integer!");
   case SignedChar:       return "signed char";
   case UnsignedChar:     return "unsigned char";
   case SignedShort:      return "short";
   case UnsignedShort:    return "unsigned short";
   case SignedInt:        return "int";
   case UnsignedInt:      return "unsigned int";
   case SignedLong:       return "long int";
   case UnsignedLong:     return "long unsigned int";
   case SignedLongLong:   return "long long int";
   case UnsignedLongLong: return "long long unsigned int";
   }
 }
 
 /// getTypeConstantSuffix - Return the constant suffix for the specified
 /// integer type enum. For example, SignedLong -> "L".
 const char *TargetInfo::getTypeConstantSuffix(IntType T) const {
   switch (T) {
   default: llvm_unreachable("not an integer!");
   case SignedChar:
   case SignedShort:
   case SignedInt:        return "";
   case SignedLong:       return "L";
   case SignedLongLong:   return "LL";
   case UnsignedChar:
     if (getCharWidth() < getIntWidth())
       return "";
     LLVM_FALLTHROUGH;
   case UnsignedShort:
     if (getShortWidth() < getIntWidth())
       return "";
     LLVM_FALLTHROUGH;
   case UnsignedInt:      return "U";
   case UnsignedLong:     return "UL";
   case UnsignedLongLong: return "ULL";
   }
 }
 
 /// getTypeFormatModifier - Return the printf format modifier for the
 /// specified integer type enum. For example, SignedLong -> "l".
 
 const char *TargetInfo::getTypeFormatModifier(IntType T) {
   switch (T) {
   default: llvm_unreachable("not an integer!");
   case SignedChar:
   case UnsignedChar:     return "hh";
   case SignedShort:
   case UnsignedShort:    return "h";
   case SignedInt:
   case UnsignedInt:      return "";
   case SignedLong:
   case UnsignedLong:     return "l";
   case SignedLongLong:
   case UnsignedLongLong: return "ll";
   }
 }
 
 /// getTypeWidth - Return the width (in bits) of the specified integer type
 /// enum. For example, SignedInt -> getIntWidth().
 unsigned TargetInfo::getTypeWidth(IntType T) const {
   switch (T) {
   default: llvm_unreachable("not an integer!");
   case SignedChar:
   case UnsignedChar:     return getCharWidth();
   case SignedShort:
   case UnsignedShort:    return getShortWidth();
   case SignedInt:
   case UnsignedInt:      return getIntWidth();
   case SignedLong:
   case UnsignedLong:     return getLongWidth();
   case SignedLongLong:
   case UnsignedLongLong: return getLongLongWidth();
   };
 }
 
 TargetInfo::IntType TargetInfo::getIntTypeByWidth(
     unsigned BitWidth, bool IsSigned) const {
   if (getCharWidth() == BitWidth)
     return IsSigned ? SignedChar : UnsignedChar;
   if (getShortWidth() == BitWidth)
     return IsSigned ? SignedShort : UnsignedShort;
   if (getIntWidth() == BitWidth)
     return IsSigned ? SignedInt : UnsignedInt;
   if (getLongWidth() == BitWidth)
     return IsSigned ? SignedLong : UnsignedLong;
   if (getLongLongWidth() == BitWidth)
     return IsSigned ? SignedLongLong : UnsignedLongLong;
   return NoInt;
 }
 
 TargetInfo::IntType TargetInfo::getLeastIntTypeByWidth(unsigned BitWidth,
                                                        bool IsSigned) const {
   if (getCharWidth() >= BitWidth)
     return IsSigned ? SignedChar : UnsignedChar;
   if (getShortWidth() >= BitWidth)
     return IsSigned ? SignedShort : UnsignedShort;
   if (getIntWidth() >= BitWidth)
     return IsSigned ? SignedInt : UnsignedInt;
   if (getLongWidth() >= BitWidth)
     return IsSigned ? SignedLong : UnsignedLong;
   if (getLongLongWidth() >= BitWidth)
     return IsSigned ? SignedLongLong : UnsignedLongLong;
   return NoInt;
 }
 
 FloatModeKind TargetInfo::getRealTypeByWidth(unsigned BitWidth,
                                              FloatModeKind ExplicitType) const {
   if (getFloatWidth() == BitWidth)
     return FloatModeKind::Float;
   if (getDoubleWidth() == BitWidth)
     return FloatModeKind::Double;
 
   switch (BitWidth) {
   case 96:
     if (&getLongDoubleFormat() == &llvm::APFloat::x87DoubleExtended())
       return FloatModeKind::LongDouble;
     break;
   case 128:
     // The caller explicitly asked for an IEEE compliant type but we still
     // have to check if the target supports it.
     if (ExplicitType == FloatModeKind::Float128)
       return hasFloat128Type() ? FloatModeKind::Float128
                                : FloatModeKind::NoFloat;
     if (ExplicitType == FloatModeKind::Ibm128)
       return hasIbm128Type() ? FloatModeKind::Ibm128
                              : FloatModeKind::NoFloat;
     if (&getLongDoubleFormat() == &llvm::APFloat::PPCDoubleDouble() ||
         &getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
       return FloatModeKind::LongDouble;
     if (hasFloat128Type())
       return FloatModeKind::Float128;
     break;
   }
 
   return FloatModeKind::NoFloat;
 }
 
 /// getTypeAlign - Return the alignment (in bits) of the specified integer type
 /// enum. For example, SignedInt -> getIntAlign().
 unsigned TargetInfo::getTypeAlign(IntType T) const {
   switch (T) {
   default: llvm_unreachable("not an integer!");
   case SignedChar:
   case UnsignedChar:     return getCharAlign();
   case SignedShort:
   case UnsignedShort:    return getShortAlign();
   case SignedInt:
   case UnsignedInt:      return getIntAlign();
   case SignedLong:
   case UnsignedLong:     return getLongAlign();
   case SignedLongLong:
   case UnsignedLongLong: return getLongLongAlign();
   };
 }
 
 /// isTypeSigned - Return whether an integer types is signed. Returns true if
 /// the type is signed; false otherwise.
 bool TargetInfo::isTypeSigned(IntType T) {
   switch (T) {
   default: llvm_unreachable("not an integer!");
   case SignedChar:
   case SignedShort:
   case SignedInt:
   case SignedLong:
   case SignedLongLong:
     return true;
   case UnsignedChar:
   case UnsignedShort:
   case UnsignedInt:
   case UnsignedLong:
   case UnsignedLongLong:
     return false;
   };
 }
 
 /// adjust - Set forced language options.
 /// Apply changes to the target information with respect to certain
 /// language options which change the target configuration and adjust
 /// the language based on the target options where applicable.
 void TargetInfo::adjust(DiagnosticsEngine &Diags, LangOptions &Opts) {
   if (Opts.NoBitFieldTypeAlign)
     UseBitFieldTypeAlignment = false;
 
   switch (Opts.WCharSize) {
   default: llvm_unreachable("invalid wchar_t width");
   case 0: break;
   case 1: WCharType = Opts.WCharIsSigned ? SignedChar : UnsignedChar; break;
   case 2: WCharType = Opts.WCharIsSigned ? SignedShort : UnsignedShort; break;
   case 4: WCharType = Opts.WCharIsSigned ? SignedInt : UnsignedInt; break;
   }
 
   if (Opts.AlignDouble) {
     DoubleAlign = LongLongAlign = 64;
     LongDoubleAlign = 64;
   }
 
   if (Opts.OpenCL) {
     // OpenCL C requires specific widths for types, irrespective of
     // what these normally are for the target.
     // We also define long long and long double here, although the
     // OpenCL standard only mentions these as "reserved".
     IntWidth = IntAlign = 32;
     LongWidth = LongAlign = 64;
     LongLongWidth = LongLongAlign = 128;
     HalfWidth = HalfAlign = 16;
     FloatWidth = FloatAlign = 32;
 
     // Embedded 32-bit targets (OpenCL EP) might have double C type
     // defined as float. Let's not override this as it might lead
     // to generating illegal code that uses 64bit doubles.
     if (DoubleWidth != FloatWidth) {
       DoubleWidth = DoubleAlign = 64;
       DoubleFormat = &llvm::APFloat::IEEEdouble();
     }
     LongDoubleWidth = LongDoubleAlign = 128;
 
     unsigned MaxPointerWidth = getMaxPointerWidth();
     assert(MaxPointerWidth == 32 || MaxPointerWidth == 64);
     bool Is32BitArch = MaxPointerWidth == 32;
     SizeType = Is32BitArch ? UnsignedInt : UnsignedLong;
     PtrDiffType = Is32BitArch ? SignedInt : SignedLong;
     IntPtrType = Is32BitArch ? SignedInt : SignedLong;
 
     IntMaxType = SignedLongLong;
     Int64Type = SignedLong;
 
     HalfFormat = &llvm::APFloat::IEEEhalf();
     FloatFormat = &llvm::APFloat::IEEEsingle();
     LongDoubleFormat = &llvm::APFloat::IEEEquad();
 
     // OpenCL C v3.0 s6.7.5 - The generic address space requires support for
     // OpenCL C 2.0 or OpenCL C 3.0 with the __opencl_c_generic_address_space
     // feature
     // OpenCL C v3.0 s6.2.1 - OpenCL pipes require support of OpenCL C 2.0
     // or later and __opencl_c_pipes feature
     // FIXME: These language options are also defined in setLangDefaults()
     // for OpenCL C 2.0 but with no access to target capabilities. Target
     // should be immutable once created and thus these language options need
     // to be defined only once.
     if (Opts.getOpenCLCompatibleVersion() == 300) {
       const auto &OpenCLFeaturesMap = getSupportedOpenCLOpts();
       Opts.OpenCLGenericAddressSpace = hasFeatureEnabled(
           OpenCLFeaturesMap, "__opencl_c_generic_address_space");
       Opts.OpenCLPipes =
           hasFeatureEnabled(OpenCLFeaturesMap, "__opencl_c_pipes");
       Opts.Blocks =
           hasFeatureEnabled(OpenCLFeaturesMap, "__opencl_c_device_enqueue");
     }
   }
 
   if (Opts.DoubleSize) {
     if (Opts.DoubleSize == 32) {
       DoubleWidth = 32;
       LongDoubleWidth = 32;
       DoubleFormat = &llvm::APFloat::IEEEsingle();
       LongDoubleFormat = &llvm::APFloat::IEEEsingle();
     } else if (Opts.DoubleSize == 64) {
       DoubleWidth = 64;
       LongDoubleWidth = 64;
       DoubleFormat = &llvm::APFloat::IEEEdouble();
       LongDoubleFormat = &llvm::APFloat::IEEEdouble();
     }
   }
 
   if (Opts.LongDoubleSize) {
     if (Opts.LongDoubleSize == DoubleWidth) {
       LongDoubleWidth = DoubleWidth;
       LongDoubleAlign = DoubleAlign;
       LongDoubleFormat = DoubleFormat;
     } else if (Opts.LongDoubleSize == 128) {
       LongDoubleWidth = LongDoubleAlign = 128;
       LongDoubleFormat = &llvm::APFloat::IEEEquad();
     }
   }
 
   if (Opts.NewAlignOverride)
     NewAlign = Opts.NewAlignOverride * getCharWidth();
 
   // Each unsigned fixed point type has the same number of fractional bits as
   // its corresponding signed type.
   PaddingOnUnsignedFixedPoint |= Opts.PaddingOnUnsignedFixedPoint;
   CheckFixedPointBits();
 
   if (Opts.ProtectParens && !checkArithmeticFenceSupported()) {
     Diags.Report(diag::err_opt_not_valid_on_target) << "-fprotect-parens";
     Opts.ProtectParens = false;
   }
 }
 
 bool TargetInfo::initFeatureMap(
     llvm::StringMap<bool> &Features, DiagnosticsEngine &Diags, StringRef CPU,
     const std::vector<std::string> &FeatureVec) const {
   for (const auto &F : FeatureVec) {
     StringRef Name = F;
     // Apply the feature via the target.
     bool Enabled = Name[0] == '+';
     setFeatureEnabled(Features, Name.substr(1), Enabled);
   }
   return true;
 }
 
 TargetInfo::CallingConvKind
 TargetInfo::getCallingConvKind(bool ClangABICompat4) const {
   if (getCXXABI() != TargetCXXABI::Microsoft &&
       (ClangABICompat4 || getTriple().getOS() == llvm::Triple::PS4))
     return CCK_ClangABI4OrPS4;
   return CCK_Default;
 }
 
 LangAS TargetInfo::getOpenCLTypeAddrSpace(OpenCLTypeKind TK) const {
   switch (TK) {
   case OCLTK_Image:
   case OCLTK_Pipe:
     return LangAS::opencl_global;
 
   case OCLTK_Sampler:
     return LangAS::opencl_constant;
 
   default:
     return LangAS::Default;
   }
 }
 
 //===----------------------------------------------------------------------===//
 
 
 static StringRef removeGCCRegisterPrefix(StringRef Name) {
   if (Name[0] == '%' || Name[0] == '#')
     Name = Name.substr(1);
 
   return Name;
 }
 
 /// isValidClobber - Returns whether the passed in string is
 /// a valid clobber in an inline asm statement. This is used by
 /// Sema.
 bool TargetInfo::isValidClobber(StringRef Name) const {
   return (isValidGCCRegisterName(Name) || Name == "memory" || Name == "cc" ||
           Name == "unwind");
 }
 
 /// isValidGCCRegisterName - Returns whether the passed in string
 /// is a valid register name according to GCC. This is used by Sema for
 /// inline asm statements.
 bool TargetInfo::isValidGCCRegisterName(StringRef Name) const {
   if (Name.empty())
     return false;
 
   // Get rid of any register prefix.
   Name = removeGCCRegisterPrefix(Name);
   if (Name.empty())
     return false;
 
   ArrayRef<const char *> Names = getGCCRegNames();
 
   // If we have a number it maps to an entry in the register name array.
   if (isDigit(Name[0])) {
     unsigned n;
     if (!Name.getAsInteger(0, n))
       return n < Names.size();
   }
 
   // Check register names.
   if (llvm::is_contained(Names, Name))
     return true;
 
   // Check any additional names that we have.
   for (const AddlRegName &ARN : getGCCAddlRegNames())
     for (const char *AN : ARN.Names) {
       if (!AN)
         break;
       // Make sure the register that the additional name is for is within
       // the bounds of the register names from above.
       if (AN == Name && ARN.RegNum < Names.size())
         return true;
     }
 
   // Now check aliases.
   for (const GCCRegAlias &GRA : getGCCRegAliases())
     for (const char *A : GRA.Aliases) {
       if (!A)
         break;
       if (A == Name)
         return true;
     }
 
   return false;
 }
 
 StringRef TargetInfo::getNormalizedGCCRegisterName(StringRef Name,
                                                    bool ReturnCanonical) const {
   assert(isValidGCCRegisterName(Name) && "Invalid register passed in");
 
   // Get rid of any register prefix.
   Name = removeGCCRegisterPrefix(Name);
 
   ArrayRef<const char *> Names = getGCCRegNames();
 
   // First, check if we have a number.
   if (isDigit(Name[0])) {
     unsigned n;
     if (!Name.getAsInteger(0, n)) {
       assert(n < Names.size() && "Out of bounds register number!");
       return Names[n];
     }
   }
 
   // Check any additional names that we have.
   for (const AddlRegName &ARN : getGCCAddlRegNames())
     for (const char *AN : ARN.Names) {
       if (!AN)
         break;
       // Make sure the register that the additional name is for is within
       // the bounds of the register names from above.
       if (AN == Name && ARN.RegNum < Names.size())
         return ReturnCanonical ? Names[ARN.RegNum] : Name;
     }
 
   // Now check aliases.
   for (const GCCRegAlias &RA : getGCCRegAliases())
     for (const char *A : RA.Aliases) {
       if (!A)
         break;
       if (A == Name)
         return RA.Register;
     }
 
   return Name;
 }
 
 bool TargetInfo::validateOutputConstraint(ConstraintInfo &Info) const {
   const char *Name = Info.getConstraintStr().c_str();
   // An output constraint must start with '=' or '+'
   if (*Name != '=' && *Name != '+')
     return false;
 
   if (*Name == '+')
     Info.setIsReadWrite();
 
   Name++;
   while (*Name) {
     switch (*Name) {
     default:
       if (!validateAsmConstraint(Name, Info)) {
         // FIXME: We temporarily return false
         // so we can add more constraints as we hit it.
         // Eventually, an unknown constraint should just be treated as 'g'.
         return false;
       }
       break;
     case '&': // early clobber.
       Info.setEarlyClobber();
       break;
     case '%': // commutative.
       // FIXME: Check that there is a another register after this one.
       break;
     case 'r': // general register.
       Info.setAllowsRegister();
       break;
     case 'm': // memory operand.
     case 'o': // offsetable memory operand.
     case 'V': // non-offsetable memory operand.
     case '<': // autodecrement memory operand.
     case '>': // autoincrement memory operand.
       Info.setAllowsMemory();
       break;
     case 'g': // general register, memory operand or immediate integer.
     case 'X': // any operand.
       Info.setAllowsRegister();
       Info.setAllowsMemory();
       break;
     case ',': // multiple alternative constraint.  Pass it.
       // Handle additional optional '=' or '+' modifiers.
       if (Name[1] == '=' || Name[1] == '+')
         Name++;
       break;
     case '#': // Ignore as constraint.
       while (Name[1] && Name[1] != ',')
         Name++;
       break;
     case '?': // Disparage slightly code.
     case '!': // Disparage severely.
     case '*': // Ignore for choosing register preferences.
     case 'i': // Ignore i,n,E,F as output constraints (match from the other
               // chars)
     case 'n':
     case 'E':
     case 'F':
       break;  // Pass them.
     }
 
     Name++;
   }
 
   // Early clobber with a read-write constraint which doesn't permit registers
   // is invalid.
   if (Info.earlyClobber() && Info.isReadWrite() && !Info.allowsRegister())
     return false;
 
   // If a constraint allows neither memory nor register operands it contains
   // only modifiers. Reject it.
   return Info.allowsMemory() || Info.allowsRegister();
 }
 
 bool TargetInfo::resolveSymbolicName(const char *&Name,
                                      ArrayRef<ConstraintInfo> OutputConstraints,
                                      unsigned &Index) const {
   assert(*Name == '[' && "Symbolic name did not start with '['");
   Name++;
   const char *Start = Name;
   while (*Name && *Name != ']')
     Name++;
 
   if (!*Name) {
     // Missing ']'
     return false;
   }
 
   std::string SymbolicName(Start, Name - Start);
 
   for (Index = 0; Index != OutputConstraints.size(); ++Index)
     if (SymbolicName == OutputConstraints[Index].getName())
       return true;
 
   return false;
 }
 
 bool TargetInfo::validateInputConstraint(
                               MutableArrayRef<ConstraintInfo> OutputConstraints,
                               ConstraintInfo &Info) const {
   const char *Name = Info.ConstraintStr.c_str();
 
   if (!*Name)
     return false;
 
   while (*Name) {
     switch (*Name) {
     default:
       // Check if we have a matching constraint
       if (*Name >= '0' && *Name <= '9') {
         const char *DigitStart = Name;
         while (Name[1] >= '0' && Name[1] <= '9')
           Name++;
         const char *DigitEnd = Name;
         unsigned i;
         if (StringRef(DigitStart, DigitEnd - DigitStart + 1)
                 .getAsInteger(10, i))
           return false;
 
         // Check if matching constraint is out of bounds.
         if (i >= OutputConstraints.size()) return false;
 
         // A number must refer to an output only operand.
         if (OutputConstraints[i].isReadWrite())
           return false;
 
         // If the constraint is already tied, it must be tied to the
         // same operand referenced to by the number.
         if (Info.hasTiedOperand() && Info.getTiedOperand() != i)
           return false;
 
         // The constraint should have the same info as the respective
         // output constraint.
         Info.setTiedOperand(i, OutputConstraints[i]);
       } else if (!validateAsmConstraint(Name, Info)) {
         // FIXME: This error return is in place temporarily so we can
         // add more constraints as we hit it.  Eventually, an unknown
         // constraint should just be treated as 'g'.
         return false;
       }
       break;
     case '[': {
       unsigned Index = 0;
       if (!resolveSymbolicName(Name, OutputConstraints, Index))
         return false;
 
       // If the constraint is already tied, it must be tied to the
       // same operand referenced to by the number.
       if (Info.hasTiedOperand() && Info.getTiedOperand() != Index)
         return false;
 
       // A number must refer to an output only operand.
       if (OutputConstraints[Index].isReadWrite())
         return false;
 
       Info.setTiedOperand(Index, OutputConstraints[Index]);
       break;
     }
     case '%': // commutative
       // FIXME: Fail if % is used with the last operand.
       break;
     case 'i': // immediate integer.
       break;
     case 'n': // immediate integer with a known value.
       Info.setRequiresImmediate();
       break;
     case 'I':  // Various constant constraints with target-specific meanings.
     case 'J':
     case 'K':
     case 'L':
     case 'M':
     case 'N':
     case 'O':
     case 'P':
       if (!validateAsmConstraint(Name, Info))
         return false;
       break;
     case 'r': // general register.
       Info.setAllowsRegister();
       break;
     case 'm': // memory operand.
     case 'o': // offsettable memory operand.
     case 'V': // non-offsettable memory operand.
     case '<': // autodecrement memory operand.
     case '>': // autoincrement memory operand.
       Info.setAllowsMemory();
       break;
     case 'g': // general register, memory operand or immediate integer.
     case 'X': // any operand.
       Info.setAllowsRegister();
       Info.setAllowsMemory();
       break;
     case 'E': // immediate floating point.
     case 'F': // immediate floating point.
     case 'p': // address operand.
       break;
     case ',': // multiple alternative constraint.  Ignore comma.
       break;
     case '#': // Ignore as constraint.
       while (Name[1] && Name[1] != ',')
         Name++;
       break;
     case '?': // Disparage slightly code.
     case '!': // Disparage severely.
     case '*': // Ignore for choosing register preferences.
       break;  // Pass them.
     }
 
     Name++;
   }
 
   return true;
 }
 
 void TargetInfo::CheckFixedPointBits() const {
   // Check that the number of fractional and integral bits (and maybe sign) can
   // fit into the bits given for a fixed point type.
   assert(ShortAccumScale + getShortAccumIBits() + 1 <= ShortAccumWidth);
   assert(AccumScale + getAccumIBits() + 1 <= AccumWidth);
   assert(LongAccumScale + getLongAccumIBits() + 1 <= LongAccumWidth);
   assert(getUnsignedShortAccumScale() + getUnsignedShortAccumIBits() <=
          ShortAccumWidth);
   assert(getUnsignedAccumScale() + getUnsignedAccumIBits() <= AccumWidth);
   assert(getUnsignedLongAccumScale() + getUnsignedLongAccumIBits() <=
          LongAccumWidth);
 
   assert(getShortFractScale() + 1 <= ShortFractWidth);
   assert(getFractScale() + 1 <= FractWidth);
   assert(getLongFractScale() + 1 <= LongFractWidth);
   assert(getUnsignedShortFractScale() <= ShortFractWidth);
   assert(getUnsignedFractScale() <= FractWidth);
   assert(getUnsignedLongFractScale() <= LongFractWidth);
 
   // Each unsigned fract type has either the same number of fractional bits
   // as, or one more fractional bit than, its corresponding signed fract type.
   assert(getShortFractScale() == getUnsignedShortFractScale() ||
          getShortFractScale() == getUnsignedShortFractScale() - 1);
   assert(getFractScale() == getUnsignedFractScale() ||
          getFractScale() == getUnsignedFractScale() - 1);
   assert(getLongFractScale() == getUnsignedLongFractScale() ||
          getLongFractScale() == getUnsignedLongFractScale() - 1);
 
   // When arranged in order of increasing rank (see 6.3.1.3a), the number of
   // fractional bits is nondecreasing for each of the following sets of
   // fixed-point types:
   // - signed fract types
   // - unsigned fract types
   // - signed accum types
   // - unsigned accum types.
   assert(getLongFractScale() >= getFractScale() &&
          getFractScale() >= getShortFractScale());
   assert(getUnsignedLongFractScale() >= getUnsignedFractScale() &&
          getUnsignedFractScale() >= getUnsignedShortFractScale());
   assert(LongAccumScale >= AccumScale && AccumScale >= ShortAccumScale);
   assert(getUnsignedLongAccumScale() >= getUnsignedAccumScale() &&
          getUnsignedAccumScale() >= getUnsignedShortAccumScale());
 
   // When arranged in order of increasing rank (see 6.3.1.3a), the number of
   // integral bits is nondecreasing for each of the following sets of
   // fixed-point types:
   // - signed accum types
   // - unsigned accum types
   assert(getLongAccumIBits() >= getAccumIBits() &&
          getAccumIBits() >= getShortAccumIBits());
   assert(getUnsignedLongAccumIBits() >= getUnsignedAccumIBits() &&
          getUnsignedAccumIBits() >= getUnsignedShortAccumIBits());
 
   // Each signed accum type has at least as many integral bits as its
   // corresponding unsigned accum type.
   assert(getShortAccumIBits() >= getUnsignedShortAccumIBits());
   assert(getAccumIBits() >= getUnsignedAccumIBits());
   assert(getLongAccumIBits() >= getUnsignedLongAccumIBits());
 }
 
 void TargetInfo::copyAuxTarget(const TargetInfo *Aux) {
   auto *Target = static_cast<TransferrableTargetInfo*>(this);
   auto *Src = static_cast<const TransferrableTargetInfo*>(Aux);
   *Target = *Src;
 }
diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGCUDANV.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGCUDANV.cpp
index c4e3f7f54f4f..414e61f25fb3 100644
--- a/contrib/llvm-project/clang/lib/CodeGen/CGCUDANV.cpp
+++ b/contrib/llvm-project/clang/lib/CodeGen/CGCUDANV.cpp
@@ -1,1157 +1,1157 @@
 //===----- CGCUDANV.cpp - Interface to NVIDIA CUDA Runtime ----------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This provides a class for CUDA code generation targeting the NVIDIA CUDA
 // runtime library.
 //
 //===----------------------------------------------------------------------===//
 
 #include "CGCUDARuntime.h"
 #include "CGCXXABI.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
 #include "clang/AST/Decl.h"
 #include "clang/Basic/Cuda.h"
 #include "clang/CodeGen/CodeGenABITypes.h"
 #include "clang/CodeGen/ConstantInitBuilder.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/ReplaceConstant.h"
 #include "llvm/Support/Format.h"
 
 using namespace clang;
 using namespace CodeGen;
 
 namespace {
 constexpr unsigned CudaFatMagic = 0x466243b1;
 constexpr unsigned HIPFatMagic = 0x48495046; // "HIPF"
 
 class CGNVCUDARuntime : public CGCUDARuntime {
 
 private:
   llvm::IntegerType *IntTy, *SizeTy;
   llvm::Type *VoidTy;
   llvm::PointerType *CharPtrTy, *VoidPtrTy, *VoidPtrPtrTy;
 
   /// Convenience reference to LLVM Context
   llvm::LLVMContext &Context;
   /// Convenience reference to the current module
   llvm::Module &TheModule;
   /// Keeps track of kernel launch stubs and handles emitted in this module
   struct KernelInfo {
     llvm::Function *Kernel; // stub function to help launch kernel
     const Decl *D;
   };
   llvm::SmallVector<KernelInfo, 16> EmittedKernels;
   // Map a device stub function to a symbol for identifying kernel in host code.
   // For CUDA, the symbol for identifying the kernel is the same as the device
   // stub function. For HIP, they are different.
   llvm::DenseMap<llvm::Function *, llvm::GlobalValue *> KernelHandles;
   // Map a kernel handle to the kernel stub.
   llvm::DenseMap<llvm::GlobalValue *, llvm::Function *> KernelStubs;
   struct VarInfo {
     llvm::GlobalVariable *Var;
     const VarDecl *D;
     DeviceVarFlags Flags;
   };
   llvm::SmallVector<VarInfo, 16> DeviceVars;
   /// Keeps track of variable containing handle of GPU binary. Populated by
   /// ModuleCtorFunction() and used to create corresponding cleanup calls in
   /// ModuleDtorFunction()
   llvm::GlobalVariable *GpuBinaryHandle = nullptr;
   /// Whether we generate relocatable device code.
   bool RelocatableDeviceCode;
   /// Mangle context for device.
   std::unique_ptr<MangleContext> DeviceMC;
 
   llvm::FunctionCallee getSetupArgumentFn() const;
   llvm::FunctionCallee getLaunchFn() const;
 
   llvm::FunctionType *getRegisterGlobalsFnTy() const;
   llvm::FunctionType *getCallbackFnTy() const;
   llvm::FunctionType *getRegisterLinkedBinaryFnTy() const;
   std::string addPrefixToName(StringRef FuncName) const;
   std::string addUnderscoredPrefixToName(StringRef FuncName) const;
 
   /// Creates a function to register all kernel stubs generated in this module.
   llvm::Function *makeRegisterGlobalsFn();
 
   /// Helper function that generates a constant string and returns a pointer to
   /// the start of the string.  The result of this function can be used anywhere
   /// where the C code specifies const char*.
   llvm::Constant *makeConstantString(const std::string &Str,
                                      const std::string &Name = "",
                                      const std::string &SectionName = "",
                                      unsigned Alignment = 0) {
     llvm::Constant *Zeros[] = {llvm::ConstantInt::get(SizeTy, 0),
                                llvm::ConstantInt::get(SizeTy, 0)};
     auto ConstStr = CGM.GetAddrOfConstantCString(Str, Name.c_str());
     llvm::GlobalVariable *GV =
         cast<llvm::GlobalVariable>(ConstStr.getPointer());
     if (!SectionName.empty()) {
       GV->setSection(SectionName);
       // Mark the address as used which make sure that this section isn't
       // merged and we will really have it in the object file.
       GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::None);
     }
     if (Alignment)
       GV->setAlignment(llvm::Align(Alignment));
 
     return llvm::ConstantExpr::getGetElementPtr(ConstStr.getElementType(),
                                                 ConstStr.getPointer(), Zeros);
   }
 
   /// Helper function that generates an empty dummy function returning void.
   llvm::Function *makeDummyFunction(llvm::FunctionType *FnTy) {
     assert(FnTy->getReturnType()->isVoidTy() &&
            "Can only generate dummy functions returning void!");
     llvm::Function *DummyFunc = llvm::Function::Create(
         FnTy, llvm::GlobalValue::InternalLinkage, "dummy", &TheModule);
 
     llvm::BasicBlock *DummyBlock =
         llvm::BasicBlock::Create(Context, "", DummyFunc);
     CGBuilderTy FuncBuilder(CGM, Context);
     FuncBuilder.SetInsertPoint(DummyBlock);
     FuncBuilder.CreateRetVoid();
 
     return DummyFunc;
   }
 
   void emitDeviceStubBodyLegacy(CodeGenFunction &CGF, FunctionArgList &Args);
   void emitDeviceStubBodyNew(CodeGenFunction &CGF, FunctionArgList &Args);
   std::string getDeviceSideName(const NamedDecl *ND) override;
 
   void registerDeviceVar(const VarDecl *VD, llvm::GlobalVariable &Var,
                          bool Extern, bool Constant) {
     DeviceVars.push_back({&Var,
                           VD,
                           {DeviceVarFlags::Variable, Extern, Constant,
                            VD->hasAttr<HIPManagedAttr>(),
                            /*Normalized*/ false, 0}});
   }
   void registerDeviceSurf(const VarDecl *VD, llvm::GlobalVariable &Var,
                           bool Extern, int Type) {
     DeviceVars.push_back({&Var,
                           VD,
                           {DeviceVarFlags::Surface, Extern, /*Constant*/ false,
                            /*Managed*/ false,
                            /*Normalized*/ false, Type}});
   }
   void registerDeviceTex(const VarDecl *VD, llvm::GlobalVariable &Var,
                          bool Extern, int Type, bool Normalized) {
     DeviceVars.push_back({&Var,
                           VD,
                           {DeviceVarFlags::Texture, Extern, /*Constant*/ false,
                            /*Managed*/ false, Normalized, Type}});
   }
 
   /// Creates module constructor function
   llvm::Function *makeModuleCtorFunction();
   /// Creates module destructor function
   llvm::Function *makeModuleDtorFunction();
   /// Transform managed variables for device compilation.
   void transformManagedVars();
 
 public:
   CGNVCUDARuntime(CodeGenModule &CGM);
 
   llvm::GlobalValue *getKernelHandle(llvm::Function *F, GlobalDecl GD) override;
   llvm::Function *getKernelStub(llvm::GlobalValue *Handle) override {
     auto Loc = KernelStubs.find(Handle);
     assert(Loc != KernelStubs.end());
     return Loc->second;
   }
   void emitDeviceStub(CodeGenFunction &CGF, FunctionArgList &Args) override;
   void handleVarRegistration(const VarDecl *VD,
                              llvm::GlobalVariable &Var) override;
   void
   internalizeDeviceSideVar(const VarDecl *D,
                            llvm::GlobalValue::LinkageTypes &Linkage) override;
 
   llvm::Function *finalizeModule() override;
 };
 
 } // end anonymous namespace
 
 std::string CGNVCUDARuntime::addPrefixToName(StringRef FuncName) const {
   if (CGM.getLangOpts().HIP)
     return ((Twine("hip") + Twine(FuncName)).str());
   return ((Twine("cuda") + Twine(FuncName)).str());
 }
 std::string
 CGNVCUDARuntime::addUnderscoredPrefixToName(StringRef FuncName) const {
   if (CGM.getLangOpts().HIP)
     return ((Twine("__hip") + Twine(FuncName)).str());
   return ((Twine("__cuda") + Twine(FuncName)).str());
 }
 
 static std::unique_ptr<MangleContext> InitDeviceMC(CodeGenModule &CGM) {
   // If the host and device have different C++ ABIs, mark it as the device
   // mangle context so that the mangling needs to retrieve the additional
   // device lambda mangling number instead of the regular host one.
   if (CGM.getContext().getAuxTargetInfo() &&
       CGM.getContext().getTargetInfo().getCXXABI().isMicrosoft() &&
       CGM.getContext().getAuxTargetInfo()->getCXXABI().isItaniumFamily()) {
     return std::unique_ptr<MangleContext>(
         CGM.getContext().createDeviceMangleContext(
             *CGM.getContext().getAuxTargetInfo()));
   }
 
   return std::unique_ptr<MangleContext>(CGM.getContext().createMangleContext(
       CGM.getContext().getAuxTargetInfo()));
 }
 
 CGNVCUDARuntime::CGNVCUDARuntime(CodeGenModule &CGM)
     : CGCUDARuntime(CGM), Context(CGM.getLLVMContext()),
       TheModule(CGM.getModule()),
       RelocatableDeviceCode(CGM.getLangOpts().GPURelocatableDeviceCode),
       DeviceMC(InitDeviceMC(CGM)) {
   CodeGen::CodeGenTypes &Types = CGM.getTypes();
   ASTContext &Ctx = CGM.getContext();
 
   IntTy = CGM.IntTy;
   SizeTy = CGM.SizeTy;
   VoidTy = CGM.VoidTy;
 
   CharPtrTy = llvm::PointerType::getUnqual(Types.ConvertType(Ctx.CharTy));
   VoidPtrTy = cast<llvm::PointerType>(Types.ConvertType(Ctx.VoidPtrTy));
   VoidPtrPtrTy = VoidPtrTy->getPointerTo();
 }
 
 llvm::FunctionCallee CGNVCUDARuntime::getSetupArgumentFn() const {
   // cudaError_t cudaSetupArgument(void *, size_t, size_t)
   llvm::Type *Params[] = {VoidPtrTy, SizeTy, SizeTy};
   return CGM.CreateRuntimeFunction(
       llvm::FunctionType::get(IntTy, Params, false),
       addPrefixToName("SetupArgument"));
 }
 
 llvm::FunctionCallee CGNVCUDARuntime::getLaunchFn() const {
   if (CGM.getLangOpts().HIP) {
     // hipError_t hipLaunchByPtr(char *);
     return CGM.CreateRuntimeFunction(
         llvm::FunctionType::get(IntTy, CharPtrTy, false), "hipLaunchByPtr");
   }
   // cudaError_t cudaLaunch(char *);
   return CGM.CreateRuntimeFunction(
       llvm::FunctionType::get(IntTy, CharPtrTy, false), "cudaLaunch");
 }
 
 llvm::FunctionType *CGNVCUDARuntime::getRegisterGlobalsFnTy() const {
   return llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false);
 }
 
 llvm::FunctionType *CGNVCUDARuntime::getCallbackFnTy() const {
   return llvm::FunctionType::get(VoidTy, VoidPtrTy, false);
 }
 
 llvm::FunctionType *CGNVCUDARuntime::getRegisterLinkedBinaryFnTy() const {
   auto *CallbackFnTy = getCallbackFnTy();
   auto *RegisterGlobalsFnTy = getRegisterGlobalsFnTy();
   llvm::Type *Params[] = {RegisterGlobalsFnTy->getPointerTo(), VoidPtrTy,
                           VoidPtrTy, CallbackFnTy->getPointerTo()};
   return llvm::FunctionType::get(VoidTy, Params, false);
 }
 
 std::string CGNVCUDARuntime::getDeviceSideName(const NamedDecl *ND) {
   GlobalDecl GD;
   // D could be either a kernel or a variable.
   if (auto *FD = dyn_cast<FunctionDecl>(ND))
     GD = GlobalDecl(FD, KernelReferenceKind::Kernel);
   else
     GD = GlobalDecl(ND);
   std::string DeviceSideName;
   MangleContext *MC;
   if (CGM.getLangOpts().CUDAIsDevice)
     MC = &CGM.getCXXABI().getMangleContext();
   else
     MC = DeviceMC.get();
   if (MC->shouldMangleDeclName(ND)) {
     SmallString<256> Buffer;
     llvm::raw_svector_ostream Out(Buffer);
     MC->mangleName(GD, Out);
     DeviceSideName = std::string(Out.str());
   } else
     DeviceSideName = std::string(ND->getIdentifier()->getName());
 
   // Make unique name for device side static file-scope variable for HIP.
   if (CGM.getContext().shouldExternalizeStaticVar(ND) &&
       CGM.getLangOpts().GPURelocatableDeviceCode &&
       !CGM.getLangOpts().CUID.empty()) {
     SmallString<256> Buffer;
     llvm::raw_svector_ostream Out(Buffer);
     Out << DeviceSideName;
-    CGM.printPostfixForExternalizedStaticVar(Out);
+    CGM.printPostfixForExternalizedDecl(Out, ND);
     DeviceSideName = std::string(Out.str());
   }
   return DeviceSideName;
 }
 
 void CGNVCUDARuntime::emitDeviceStub(CodeGenFunction &CGF,
                                      FunctionArgList &Args) {
   EmittedKernels.push_back({CGF.CurFn, CGF.CurFuncDecl});
   if (auto *GV = dyn_cast<llvm::GlobalVariable>(KernelHandles[CGF.CurFn])) {
     GV->setLinkage(CGF.CurFn->getLinkage());
     GV->setInitializer(CGF.CurFn);
   }
   if (CudaFeatureEnabled(CGM.getTarget().getSDKVersion(),
                          CudaFeature::CUDA_USES_NEW_LAUNCH) ||
       (CGF.getLangOpts().HIP && CGF.getLangOpts().HIPUseNewLaunchAPI))
     emitDeviceStubBodyNew(CGF, Args);
   else
     emitDeviceStubBodyLegacy(CGF, Args);
 }
 
 // CUDA 9.0+ uses new way to launch kernels. Parameters are packed in a local
 // array and kernels are launched using cudaLaunchKernel().
 void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
                                             FunctionArgList &Args) {
   // Build the shadow stack entry at the very start of the function.
 
   // Calculate amount of space we will need for all arguments.  If we have no
   // args, allocate a single pointer so we still have a valid pointer to the
   // argument array that we can pass to runtime, even if it will be unused.
   Address KernelArgs = CGF.CreateTempAlloca(
       VoidPtrTy, CharUnits::fromQuantity(16), "kernel_args",
       llvm::ConstantInt::get(SizeTy, std::max<size_t>(1, Args.size())));
   // Store pointers to the arguments in a locally allocated launch_args.
   for (unsigned i = 0; i < Args.size(); ++i) {
     llvm::Value* VarPtr = CGF.GetAddrOfLocalVar(Args[i]).getPointer();
     llvm::Value *VoidVarPtr = CGF.Builder.CreatePointerCast(VarPtr, VoidPtrTy);
     CGF.Builder.CreateDefaultAlignedStore(
         VoidVarPtr,
         CGF.Builder.CreateConstGEP1_32(VoidPtrTy, KernelArgs.getPointer(), i));
   }
 
   llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end");
 
   // Lookup cudaLaunchKernel/hipLaunchKernel function.
   // cudaError_t cudaLaunchKernel(const void *func, dim3 gridDim, dim3 blockDim,
   //                              void **args, size_t sharedMem,
   //                              cudaStream_t stream);
   // hipError_t hipLaunchKernel(const void *func, dim3 gridDim, dim3 blockDim,
   //                            void **args, size_t sharedMem,
   //                            hipStream_t stream);
   TranslationUnitDecl *TUDecl = CGM.getContext().getTranslationUnitDecl();
   DeclContext *DC = TranslationUnitDecl::castToDeclContext(TUDecl);
   auto LaunchKernelName = addPrefixToName("LaunchKernel");
   IdentifierInfo &cudaLaunchKernelII =
       CGM.getContext().Idents.get(LaunchKernelName);
   FunctionDecl *cudaLaunchKernelFD = nullptr;
   for (auto *Result : DC->lookup(&cudaLaunchKernelII)) {
     if (FunctionDecl *FD = dyn_cast<FunctionDecl>(Result))
       cudaLaunchKernelFD = FD;
   }
 
   if (cudaLaunchKernelFD == nullptr) {
     CGM.Error(CGF.CurFuncDecl->getLocation(),
               "Can't find declaration for " + LaunchKernelName);
     return;
   }
   // Create temporary dim3 grid_dim, block_dim.
   ParmVarDecl *GridDimParam = cudaLaunchKernelFD->getParamDecl(1);
   QualType Dim3Ty = GridDimParam->getType();
   Address GridDim =
       CGF.CreateMemTemp(Dim3Ty, CharUnits::fromQuantity(8), "grid_dim");
   Address BlockDim =
       CGF.CreateMemTemp(Dim3Ty, CharUnits::fromQuantity(8), "block_dim");
   Address ShmemSize =
       CGF.CreateTempAlloca(SizeTy, CGM.getSizeAlign(), "shmem_size");
   Address Stream =
       CGF.CreateTempAlloca(VoidPtrTy, CGM.getPointerAlign(), "stream");
   llvm::FunctionCallee cudaPopConfigFn = CGM.CreateRuntimeFunction(
       llvm::FunctionType::get(IntTy,
                               {/*gridDim=*/GridDim.getType(),
                                /*blockDim=*/BlockDim.getType(),
                                /*ShmemSize=*/ShmemSize.getType(),
                                /*Stream=*/Stream.getType()},
                               /*isVarArg=*/false),
       addUnderscoredPrefixToName("PopCallConfiguration"));
 
   CGF.EmitRuntimeCallOrInvoke(cudaPopConfigFn,
                               {GridDim.getPointer(), BlockDim.getPointer(),
                                ShmemSize.getPointer(), Stream.getPointer()});
 
   // Emit the call to cudaLaunch
   llvm::Value *Kernel =
       CGF.Builder.CreatePointerCast(KernelHandles[CGF.CurFn], VoidPtrTy);
   CallArgList LaunchKernelArgs;
   LaunchKernelArgs.add(RValue::get(Kernel),
                        cudaLaunchKernelFD->getParamDecl(0)->getType());
   LaunchKernelArgs.add(RValue::getAggregate(GridDim), Dim3Ty);
   LaunchKernelArgs.add(RValue::getAggregate(BlockDim), Dim3Ty);
   LaunchKernelArgs.add(RValue::get(KernelArgs.getPointer()),
                        cudaLaunchKernelFD->getParamDecl(3)->getType());
   LaunchKernelArgs.add(RValue::get(CGF.Builder.CreateLoad(ShmemSize)),
                        cudaLaunchKernelFD->getParamDecl(4)->getType());
   LaunchKernelArgs.add(RValue::get(CGF.Builder.CreateLoad(Stream)),
                        cudaLaunchKernelFD->getParamDecl(5)->getType());
 
   QualType QT = cudaLaunchKernelFD->getType();
   QualType CQT = QT.getCanonicalType();
   llvm::Type *Ty = CGM.getTypes().ConvertType(CQT);
   llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
 
   const CGFunctionInfo &FI =
       CGM.getTypes().arrangeFunctionDeclaration(cudaLaunchKernelFD);
   llvm::FunctionCallee cudaLaunchKernelFn =
       CGM.CreateRuntimeFunction(FTy, LaunchKernelName);
   CGF.EmitCall(FI, CGCallee::forDirect(cudaLaunchKernelFn), ReturnValueSlot(),
                LaunchKernelArgs);
   CGF.EmitBranch(EndBlock);
 
   CGF.EmitBlock(EndBlock);
 }
 
 void CGNVCUDARuntime::emitDeviceStubBodyLegacy(CodeGenFunction &CGF,
                                                FunctionArgList &Args) {
   // Emit a call to cudaSetupArgument for each arg in Args.
   llvm::FunctionCallee cudaSetupArgFn = getSetupArgumentFn();
   llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end");
   CharUnits Offset = CharUnits::Zero();
   for (const VarDecl *A : Args) {
     auto TInfo = CGM.getContext().getTypeInfoInChars(A->getType());
     Offset = Offset.alignTo(TInfo.Align);
     llvm::Value *Args[] = {
         CGF.Builder.CreatePointerCast(CGF.GetAddrOfLocalVar(A).getPointer(),
                                       VoidPtrTy),
         llvm::ConstantInt::get(SizeTy, TInfo.Width.getQuantity()),
         llvm::ConstantInt::get(SizeTy, Offset.getQuantity()),
     };
     llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(cudaSetupArgFn, Args);
     llvm::Constant *Zero = llvm::ConstantInt::get(IntTy, 0);
     llvm::Value *CBZero = CGF.Builder.CreateICmpEQ(CB, Zero);
     llvm::BasicBlock *NextBlock = CGF.createBasicBlock("setup.next");
     CGF.Builder.CreateCondBr(CBZero, NextBlock, EndBlock);
     CGF.EmitBlock(NextBlock);
     Offset += TInfo.Width;
   }
 
   // Emit the call to cudaLaunch
   llvm::FunctionCallee cudaLaunchFn = getLaunchFn();
   llvm::Value *Arg =
       CGF.Builder.CreatePointerCast(KernelHandles[CGF.CurFn], CharPtrTy);
   CGF.EmitRuntimeCallOrInvoke(cudaLaunchFn, Arg);
   CGF.EmitBranch(EndBlock);
 
   CGF.EmitBlock(EndBlock);
 }
 
 // Replace the original variable Var with the address loaded from variable
 // ManagedVar populated by HIP runtime.
 static void replaceManagedVar(llvm::GlobalVariable *Var,
                               llvm::GlobalVariable *ManagedVar) {
   SmallVector<SmallVector<llvm::User *, 8>, 8> WorkList;
   for (auto &&VarUse : Var->uses()) {
     WorkList.push_back({VarUse.getUser()});
   }
   while (!WorkList.empty()) {
     auto &&WorkItem = WorkList.pop_back_val();
     auto *U = WorkItem.back();
     if (isa<llvm::ConstantExpr>(U)) {
       for (auto &&UU : U->uses()) {
         WorkItem.push_back(UU.getUser());
         WorkList.push_back(WorkItem);
         WorkItem.pop_back();
       }
       continue;
     }
     if (auto *I = dyn_cast<llvm::Instruction>(U)) {
       llvm::Value *OldV = Var;
       llvm::Instruction *NewV =
           new llvm::LoadInst(Var->getType(), ManagedVar, "ld.managed", false,
                              llvm::Align(Var->getAlignment()), I);
       WorkItem.pop_back();
       // Replace constant expressions directly or indirectly using the managed
       // variable with instructions.
       for (auto &&Op : WorkItem) {
         auto *CE = cast<llvm::ConstantExpr>(Op);
         auto *NewInst = CE->getAsInstruction(I);
         NewInst->replaceUsesOfWith(OldV, NewV);
         OldV = CE;
         NewV = NewInst;
       }
       I->replaceUsesOfWith(OldV, NewV);
     } else {
       llvm_unreachable("Invalid use of managed variable");
     }
   }
 }
 
 /// Creates a function that sets up state on the host side for CUDA objects that
 /// have a presence on both the host and device sides. Specifically, registers
 /// the host side of kernel functions and device global variables with the CUDA
 /// runtime.
 /// \code
 /// void __cuda_register_globals(void** GpuBinaryHandle) {
 ///    __cudaRegisterFunction(GpuBinaryHandle,Kernel0,...);
 ///    ...
 ///    __cudaRegisterFunction(GpuBinaryHandle,KernelM,...);
 ///    __cudaRegisterVar(GpuBinaryHandle, GlobalVar0, ...);
 ///    ...
 ///    __cudaRegisterVar(GpuBinaryHandle, GlobalVarN, ...);
 /// }
 /// \endcode
 llvm::Function *CGNVCUDARuntime::makeRegisterGlobalsFn() {
   // No need to register anything
   if (EmittedKernels.empty() && DeviceVars.empty())
     return nullptr;
 
   llvm::Function *RegisterKernelsFunc = llvm::Function::Create(
       getRegisterGlobalsFnTy(), llvm::GlobalValue::InternalLinkage,
       addUnderscoredPrefixToName("_register_globals"), &TheModule);
   llvm::BasicBlock *EntryBB =
       llvm::BasicBlock::Create(Context, "entry", RegisterKernelsFunc);
   CGBuilderTy Builder(CGM, Context);
   Builder.SetInsertPoint(EntryBB);
 
   // void __cudaRegisterFunction(void **, const char *, char *, const char *,
   //                             int, uint3*, uint3*, dim3*, dim3*, int*)
   llvm::Type *RegisterFuncParams[] = {
       VoidPtrPtrTy, CharPtrTy, CharPtrTy, CharPtrTy, IntTy,
       VoidPtrTy,    VoidPtrTy, VoidPtrTy, VoidPtrTy, IntTy->getPointerTo()};
   llvm::FunctionCallee RegisterFunc = CGM.CreateRuntimeFunction(
       llvm::FunctionType::get(IntTy, RegisterFuncParams, false),
       addUnderscoredPrefixToName("RegisterFunction"));
 
   // Extract GpuBinaryHandle passed as the first argument passed to
   // __cuda_register_globals() and generate __cudaRegisterFunction() call for
   // each emitted kernel.
   llvm::Argument &GpuBinaryHandlePtr = *RegisterKernelsFunc->arg_begin();
   for (auto &&I : EmittedKernels) {
     llvm::Constant *KernelName =
         makeConstantString(getDeviceSideName(cast<NamedDecl>(I.D)));
     llvm::Constant *NullPtr = llvm::ConstantPointerNull::get(VoidPtrTy);
     llvm::Value *Args[] = {
         &GpuBinaryHandlePtr,
         Builder.CreateBitCast(KernelHandles[I.Kernel], VoidPtrTy),
         KernelName,
         KernelName,
         llvm::ConstantInt::get(IntTy, -1),
         NullPtr,
         NullPtr,
         NullPtr,
         NullPtr,
         llvm::ConstantPointerNull::get(IntTy->getPointerTo())};
     Builder.CreateCall(RegisterFunc, Args);
   }
 
   llvm::Type *VarSizeTy = IntTy;
   // For HIP or CUDA 9.0+, device variable size is type of `size_t`.
   if (CGM.getLangOpts().HIP ||
       ToCudaVersion(CGM.getTarget().getSDKVersion()) >= CudaVersion::CUDA_90)
     VarSizeTy = SizeTy;
 
   // void __cudaRegisterVar(void **, char *, char *, const char *,
   //                        int, int, int, int)
   llvm::Type *RegisterVarParams[] = {VoidPtrPtrTy, CharPtrTy, CharPtrTy,
                                      CharPtrTy,    IntTy,     VarSizeTy,
                                      IntTy,        IntTy};
   llvm::FunctionCallee RegisterVar = CGM.CreateRuntimeFunction(
       llvm::FunctionType::get(VoidTy, RegisterVarParams, false),
       addUnderscoredPrefixToName("RegisterVar"));
   // void __hipRegisterManagedVar(void **, char *, char *, const char *,
   //                              size_t, unsigned)
   llvm::Type *RegisterManagedVarParams[] = {VoidPtrPtrTy, CharPtrTy, CharPtrTy,
                                             CharPtrTy,    VarSizeTy, IntTy};
   llvm::FunctionCallee RegisterManagedVar = CGM.CreateRuntimeFunction(
       llvm::FunctionType::get(VoidTy, RegisterManagedVarParams, false),
       addUnderscoredPrefixToName("RegisterManagedVar"));
   // void __cudaRegisterSurface(void **, const struct surfaceReference *,
   //                            const void **, const char *, int, int);
   llvm::FunctionCallee RegisterSurf = CGM.CreateRuntimeFunction(
       llvm::FunctionType::get(
           VoidTy, {VoidPtrPtrTy, VoidPtrTy, CharPtrTy, CharPtrTy, IntTy, IntTy},
           false),
       addUnderscoredPrefixToName("RegisterSurface"));
   // void __cudaRegisterTexture(void **, const struct textureReference *,
   //                            const void **, const char *, int, int, int)
   llvm::FunctionCallee RegisterTex = CGM.CreateRuntimeFunction(
       llvm::FunctionType::get(
           VoidTy,
           {VoidPtrPtrTy, VoidPtrTy, CharPtrTy, CharPtrTy, IntTy, IntTy, IntTy},
           false),
       addUnderscoredPrefixToName("RegisterTexture"));
   for (auto &&Info : DeviceVars) {
     llvm::GlobalVariable *Var = Info.Var;
     assert((!Var->isDeclaration() || Info.Flags.isManaged()) &&
            "External variables should not show up here, except HIP managed "
            "variables");
     llvm::Constant *VarName = makeConstantString(getDeviceSideName(Info.D));
     switch (Info.Flags.getKind()) {
     case DeviceVarFlags::Variable: {
       uint64_t VarSize =
           CGM.getDataLayout().getTypeAllocSize(Var->getValueType());
       if (Info.Flags.isManaged()) {
         auto *ManagedVar = new llvm::GlobalVariable(
             CGM.getModule(), Var->getType(),
             /*isConstant=*/false, Var->getLinkage(),
             /*Init=*/Var->isDeclaration()
                 ? nullptr
                 : llvm::ConstantPointerNull::get(Var->getType()),
             /*Name=*/"", /*InsertBefore=*/nullptr,
             llvm::GlobalVariable::NotThreadLocal);
         ManagedVar->setDSOLocal(Var->isDSOLocal());
         ManagedVar->setVisibility(Var->getVisibility());
         ManagedVar->setExternallyInitialized(true);
         ManagedVar->takeName(Var);
         Var->setName(Twine(ManagedVar->getName() + ".managed"));
         replaceManagedVar(Var, ManagedVar);
         llvm::Value *Args[] = {
             &GpuBinaryHandlePtr,
             Builder.CreateBitCast(ManagedVar, VoidPtrTy),
             Builder.CreateBitCast(Var, VoidPtrTy),
             VarName,
             llvm::ConstantInt::get(VarSizeTy, VarSize),
             llvm::ConstantInt::get(IntTy, Var->getAlignment())};
         if (!Var->isDeclaration())
           Builder.CreateCall(RegisterManagedVar, Args);
       } else {
         llvm::Value *Args[] = {
             &GpuBinaryHandlePtr,
             Builder.CreateBitCast(Var, VoidPtrTy),
             VarName,
             VarName,
             llvm::ConstantInt::get(IntTy, Info.Flags.isExtern()),
             llvm::ConstantInt::get(VarSizeTy, VarSize),
             llvm::ConstantInt::get(IntTy, Info.Flags.isConstant()),
             llvm::ConstantInt::get(IntTy, 0)};
         Builder.CreateCall(RegisterVar, Args);
       }
       break;
     }
     case DeviceVarFlags::Surface:
       Builder.CreateCall(
           RegisterSurf,
           {&GpuBinaryHandlePtr, Builder.CreateBitCast(Var, VoidPtrTy), VarName,
            VarName, llvm::ConstantInt::get(IntTy, Info.Flags.getSurfTexType()),
            llvm::ConstantInt::get(IntTy, Info.Flags.isExtern())});
       break;
     case DeviceVarFlags::Texture:
       Builder.CreateCall(
           RegisterTex,
           {&GpuBinaryHandlePtr, Builder.CreateBitCast(Var, VoidPtrTy), VarName,
            VarName, llvm::ConstantInt::get(IntTy, Info.Flags.getSurfTexType()),
            llvm::ConstantInt::get(IntTy, Info.Flags.isNormalized()),
            llvm::ConstantInt::get(IntTy, Info.Flags.isExtern())});
       break;
     }
   }
 
   Builder.CreateRetVoid();
   return RegisterKernelsFunc;
 }
 
 /// Creates a global constructor function for the module:
 ///
 /// For CUDA:
 /// \code
 /// void __cuda_module_ctor(void*) {
 ///     Handle = __cudaRegisterFatBinary(GpuBinaryBlob);
 ///     __cuda_register_globals(Handle);
 /// }
 /// \endcode
 ///
 /// For HIP:
 /// \code
 /// void __hip_module_ctor(void*) {
 ///     if (__hip_gpubin_handle == 0) {
 ///         __hip_gpubin_handle  = __hipRegisterFatBinary(GpuBinaryBlob);
 ///         __hip_register_globals(__hip_gpubin_handle);
 ///     }
 /// }
 /// \endcode
 llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() {
   bool IsHIP = CGM.getLangOpts().HIP;
   bool IsCUDA = CGM.getLangOpts().CUDA;
   // No need to generate ctors/dtors if there is no GPU binary.
   StringRef CudaGpuBinaryFileName = CGM.getCodeGenOpts().CudaGpuBinaryFileName;
   if (CudaGpuBinaryFileName.empty() && !IsHIP)
     return nullptr;
   if ((IsHIP || (IsCUDA && !RelocatableDeviceCode)) && EmittedKernels.empty() &&
       DeviceVars.empty())
     return nullptr;
 
   // void __{cuda|hip}_register_globals(void* handle);
   llvm::Function *RegisterGlobalsFunc = makeRegisterGlobalsFn();
   // We always need a function to pass in as callback. Create a dummy
   // implementation if we don't need to register anything.
   if (RelocatableDeviceCode && !RegisterGlobalsFunc)
     RegisterGlobalsFunc = makeDummyFunction(getRegisterGlobalsFnTy());
 
   // void ** __{cuda|hip}RegisterFatBinary(void *);
   llvm::FunctionCallee RegisterFatbinFunc = CGM.CreateRuntimeFunction(
       llvm::FunctionType::get(VoidPtrPtrTy, VoidPtrTy, false),
       addUnderscoredPrefixToName("RegisterFatBinary"));
   // struct { int magic, int version, void * gpu_binary, void * dont_care };
   llvm::StructType *FatbinWrapperTy =
       llvm::StructType::get(IntTy, IntTy, VoidPtrTy, VoidPtrTy);
 
   // Register GPU binary with the CUDA runtime, store returned handle in a
   // global variable and save a reference in GpuBinaryHandle to be cleaned up
   // in destructor on exit. Then associate all known kernels with the GPU binary
   // handle so CUDA runtime can figure out what to call on the GPU side.
   std::unique_ptr<llvm::MemoryBuffer> CudaGpuBinary = nullptr;
   if (!CudaGpuBinaryFileName.empty()) {
     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> CudaGpuBinaryOrErr =
         llvm::MemoryBuffer::getFileOrSTDIN(CudaGpuBinaryFileName);
     if (std::error_code EC = CudaGpuBinaryOrErr.getError()) {
       CGM.getDiags().Report(diag::err_cannot_open_file)
           << CudaGpuBinaryFileName << EC.message();
       return nullptr;
     }
     CudaGpuBinary = std::move(CudaGpuBinaryOrErr.get());
   }
 
   llvm::Function *ModuleCtorFunc = llvm::Function::Create(
       llvm::FunctionType::get(VoidTy, VoidPtrTy, false),
       llvm::GlobalValue::InternalLinkage,
       addUnderscoredPrefixToName("_module_ctor"), &TheModule);
   llvm::BasicBlock *CtorEntryBB =
       llvm::BasicBlock::Create(Context, "entry", ModuleCtorFunc);
   CGBuilderTy CtorBuilder(CGM, Context);
 
   CtorBuilder.SetInsertPoint(CtorEntryBB);
 
   const char *FatbinConstantName;
   const char *FatbinSectionName;
   const char *ModuleIDSectionName;
   StringRef ModuleIDPrefix;
   llvm::Constant *FatBinStr;
   unsigned FatMagic;
   if (IsHIP) {
     FatbinConstantName = ".hip_fatbin";
     FatbinSectionName = ".hipFatBinSegment";
 
     ModuleIDSectionName = "__hip_module_id";
     ModuleIDPrefix = "__hip_";
 
     if (CudaGpuBinary) {
       // If fatbin is available from early finalization, create a string
       // literal containing the fat binary loaded from the given file.
       const unsigned HIPCodeObjectAlign = 4096;
       FatBinStr =
           makeConstantString(std::string(CudaGpuBinary->getBuffer()), "",
                              FatbinConstantName, HIPCodeObjectAlign);
     } else {
       // If fatbin is not available, create an external symbol
       // __hip_fatbin in section .hip_fatbin. The external symbol is supposed
       // to contain the fat binary but will be populated somewhere else,
       // e.g. by lld through link script.
       FatBinStr = new llvm::GlobalVariable(
         CGM.getModule(), CGM.Int8Ty,
         /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, nullptr,
         "__hip_fatbin", nullptr,
         llvm::GlobalVariable::NotThreadLocal);
       cast<llvm::GlobalVariable>(FatBinStr)->setSection(FatbinConstantName);
     }
 
     FatMagic = HIPFatMagic;
   } else {
     if (RelocatableDeviceCode)
       FatbinConstantName = CGM.getTriple().isMacOSX()
                                ? "__NV_CUDA,__nv_relfatbin"
                                : "__nv_relfatbin";
     else
       FatbinConstantName =
           CGM.getTriple().isMacOSX() ? "__NV_CUDA,__nv_fatbin" : ".nv_fatbin";
     // NVIDIA's cuobjdump looks for fatbins in this section.
     FatbinSectionName =
         CGM.getTriple().isMacOSX() ? "__NV_CUDA,__fatbin" : ".nvFatBinSegment";
 
     ModuleIDSectionName = CGM.getTriple().isMacOSX()
                               ? "__NV_CUDA,__nv_module_id"
                               : "__nv_module_id";
     ModuleIDPrefix = "__nv_";
 
     // For CUDA, create a string literal containing the fat binary loaded from
     // the given file.
     FatBinStr = makeConstantString(std::string(CudaGpuBinary->getBuffer()), "",
                                    FatbinConstantName, 8);
     FatMagic = CudaFatMagic;
   }
 
   // Create initialized wrapper structure that points to the loaded GPU binary
   ConstantInitBuilder Builder(CGM);
   auto Values = Builder.beginStruct(FatbinWrapperTy);
   // Fatbin wrapper magic.
   Values.addInt(IntTy, FatMagic);
   // Fatbin version.
   Values.addInt(IntTy, 1);
   // Data.
   Values.add(FatBinStr);
   // Unused in fatbin v1.
   Values.add(llvm::ConstantPointerNull::get(VoidPtrTy));
   llvm::GlobalVariable *FatbinWrapper = Values.finishAndCreateGlobal(
       addUnderscoredPrefixToName("_fatbin_wrapper"), CGM.getPointerAlign(),
       /*constant*/ true);
   FatbinWrapper->setSection(FatbinSectionName);
 
   // There is only one HIP fat binary per linked module, however there are
   // multiple constructor functions. Make sure the fat binary is registered
   // only once. The constructor functions are executed by the dynamic loader
   // before the program gains control. The dynamic loader cannot execute the
   // constructor functions concurrently since doing that would not guarantee
   // thread safety of the loaded program. Therefore we can assume sequential
   // execution of constructor functions here.
   if (IsHIP) {
     auto Linkage = CudaGpuBinary ? llvm::GlobalValue::InternalLinkage :
         llvm::GlobalValue::LinkOnceAnyLinkage;
     llvm::BasicBlock *IfBlock =
         llvm::BasicBlock::Create(Context, "if", ModuleCtorFunc);
     llvm::BasicBlock *ExitBlock =
         llvm::BasicBlock::Create(Context, "exit", ModuleCtorFunc);
     // The name, size, and initialization pattern of this variable is part
     // of HIP ABI.
     GpuBinaryHandle = new llvm::GlobalVariable(
         TheModule, VoidPtrPtrTy, /*isConstant=*/false,
         Linkage,
         /*Initializer=*/llvm::ConstantPointerNull::get(VoidPtrPtrTy),
         "__hip_gpubin_handle");
     if (Linkage == llvm::GlobalValue::LinkOnceAnyLinkage)
       GpuBinaryHandle->setComdat(
           CGM.getModule().getOrInsertComdat(GpuBinaryHandle->getName()));
     GpuBinaryHandle->setAlignment(CGM.getPointerAlign().getAsAlign());
     // Prevent the weak symbol in different shared libraries being merged.
     if (Linkage != llvm::GlobalValue::InternalLinkage)
       GpuBinaryHandle->setVisibility(llvm::GlobalValue::HiddenVisibility);
     Address GpuBinaryAddr(
         GpuBinaryHandle,
         CharUnits::fromQuantity(GpuBinaryHandle->getAlignment()));
     {
       auto *HandleValue = CtorBuilder.CreateLoad(GpuBinaryAddr);
       llvm::Constant *Zero =
           llvm::Constant::getNullValue(HandleValue->getType());
       llvm::Value *EQZero = CtorBuilder.CreateICmpEQ(HandleValue, Zero);
       CtorBuilder.CreateCondBr(EQZero, IfBlock, ExitBlock);
     }
     {
       CtorBuilder.SetInsertPoint(IfBlock);
       // GpuBinaryHandle = __hipRegisterFatBinary(&FatbinWrapper);
       llvm::CallInst *RegisterFatbinCall = CtorBuilder.CreateCall(
           RegisterFatbinFunc,
           CtorBuilder.CreateBitCast(FatbinWrapper, VoidPtrTy));
       CtorBuilder.CreateStore(RegisterFatbinCall, GpuBinaryAddr);
       CtorBuilder.CreateBr(ExitBlock);
     }
     {
       CtorBuilder.SetInsertPoint(ExitBlock);
       // Call __hip_register_globals(GpuBinaryHandle);
       if (RegisterGlobalsFunc) {
         auto *HandleValue = CtorBuilder.CreateLoad(GpuBinaryAddr);
         CtorBuilder.CreateCall(RegisterGlobalsFunc, HandleValue);
       }
     }
   } else if (!RelocatableDeviceCode) {
     // Register binary with CUDA runtime. This is substantially different in
     // default mode vs. separate compilation!
     // GpuBinaryHandle = __cudaRegisterFatBinary(&FatbinWrapper);
     llvm::CallInst *RegisterFatbinCall = CtorBuilder.CreateCall(
         RegisterFatbinFunc,
         CtorBuilder.CreateBitCast(FatbinWrapper, VoidPtrTy));
     GpuBinaryHandle = new llvm::GlobalVariable(
         TheModule, VoidPtrPtrTy, false, llvm::GlobalValue::InternalLinkage,
         llvm::ConstantPointerNull::get(VoidPtrPtrTy), "__cuda_gpubin_handle");
     GpuBinaryHandle->setAlignment(CGM.getPointerAlign().getAsAlign());
     CtorBuilder.CreateAlignedStore(RegisterFatbinCall, GpuBinaryHandle,
                                    CGM.getPointerAlign());
 
     // Call __cuda_register_globals(GpuBinaryHandle);
     if (RegisterGlobalsFunc)
       CtorBuilder.CreateCall(RegisterGlobalsFunc, RegisterFatbinCall);
 
     // Call __cudaRegisterFatBinaryEnd(Handle) if this CUDA version needs it.
     if (CudaFeatureEnabled(CGM.getTarget().getSDKVersion(),
                            CudaFeature::CUDA_USES_FATBIN_REGISTER_END)) {
       // void __cudaRegisterFatBinaryEnd(void **);
       llvm::FunctionCallee RegisterFatbinEndFunc = CGM.CreateRuntimeFunction(
           llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false),
           "__cudaRegisterFatBinaryEnd");
       CtorBuilder.CreateCall(RegisterFatbinEndFunc, RegisterFatbinCall);
     }
   } else {
     // Generate a unique module ID.
     SmallString<64> ModuleID;
     llvm::raw_svector_ostream OS(ModuleID);
     OS << ModuleIDPrefix << llvm::format("%" PRIx64, FatbinWrapper->getGUID());
     llvm::Constant *ModuleIDConstant = makeConstantString(
         std::string(ModuleID.str()), "", ModuleIDSectionName, 32);
 
     // Create an alias for the FatbinWrapper that nvcc will look for.
     llvm::GlobalAlias::create(llvm::GlobalValue::ExternalLinkage,
                               Twine("__fatbinwrap") + ModuleID, FatbinWrapper);
 
     // void __cudaRegisterLinkedBinary%ModuleID%(void (*)(void *), void *,
     // void *, void (*)(void **))
     SmallString<128> RegisterLinkedBinaryName("__cudaRegisterLinkedBinary");
     RegisterLinkedBinaryName += ModuleID;
     llvm::FunctionCallee RegisterLinkedBinaryFunc = CGM.CreateRuntimeFunction(
         getRegisterLinkedBinaryFnTy(), RegisterLinkedBinaryName);
 
     assert(RegisterGlobalsFunc && "Expecting at least dummy function!");
     llvm::Value *Args[] = {RegisterGlobalsFunc,
                            CtorBuilder.CreateBitCast(FatbinWrapper, VoidPtrTy),
                            ModuleIDConstant,
                            makeDummyFunction(getCallbackFnTy())};
     CtorBuilder.CreateCall(RegisterLinkedBinaryFunc, Args);
   }
 
   // Create destructor and register it with atexit() the way NVCC does it. Doing
   // it during regular destructor phase worked in CUDA before 9.2 but results in
   // double-free in 9.2.
   if (llvm::Function *CleanupFn = makeModuleDtorFunction()) {
     // extern "C" int atexit(void (*f)(void));
     llvm::FunctionType *AtExitTy =
         llvm::FunctionType::get(IntTy, CleanupFn->getType(), false);
     llvm::FunctionCallee AtExitFunc =
         CGM.CreateRuntimeFunction(AtExitTy, "atexit", llvm::AttributeList(),
                                   /*Local=*/true);
     CtorBuilder.CreateCall(AtExitFunc, CleanupFn);
   }
 
   CtorBuilder.CreateRetVoid();
   return ModuleCtorFunc;
 }
 
 /// Creates a global destructor function that unregisters the GPU code blob
 /// registered by constructor.
 ///
 /// For CUDA:
 /// \code
 /// void __cuda_module_dtor(void*) {
 ///     __cudaUnregisterFatBinary(Handle);
 /// }
 /// \endcode
 ///
 /// For HIP:
 /// \code
 /// void __hip_module_dtor(void*) {
 ///     if (__hip_gpubin_handle) {
 ///         __hipUnregisterFatBinary(__hip_gpubin_handle);
 ///         __hip_gpubin_handle = 0;
 ///     }
 /// }
 /// \endcode
 llvm::Function *CGNVCUDARuntime::makeModuleDtorFunction() {
   // No need for destructor if we don't have a handle to unregister.
   if (!GpuBinaryHandle)
     return nullptr;
 
   // void __cudaUnregisterFatBinary(void ** handle);
   llvm::FunctionCallee UnregisterFatbinFunc = CGM.CreateRuntimeFunction(
       llvm::FunctionType::get(VoidTy, VoidPtrPtrTy, false),
       addUnderscoredPrefixToName("UnregisterFatBinary"));
 
   llvm::Function *ModuleDtorFunc = llvm::Function::Create(
       llvm::FunctionType::get(VoidTy, VoidPtrTy, false),
       llvm::GlobalValue::InternalLinkage,
       addUnderscoredPrefixToName("_module_dtor"), &TheModule);
 
   llvm::BasicBlock *DtorEntryBB =
       llvm::BasicBlock::Create(Context, "entry", ModuleDtorFunc);
   CGBuilderTy DtorBuilder(CGM, Context);
   DtorBuilder.SetInsertPoint(DtorEntryBB);
 
   Address GpuBinaryAddr(GpuBinaryHandle, CharUnits::fromQuantity(
                                              GpuBinaryHandle->getAlignment()));
   auto *HandleValue = DtorBuilder.CreateLoad(GpuBinaryAddr);
   // There is only one HIP fat binary per linked module, however there are
   // multiple destructor functions. Make sure the fat binary is unregistered
   // only once.
   if (CGM.getLangOpts().HIP) {
     llvm::BasicBlock *IfBlock =
         llvm::BasicBlock::Create(Context, "if", ModuleDtorFunc);
     llvm::BasicBlock *ExitBlock =
         llvm::BasicBlock::Create(Context, "exit", ModuleDtorFunc);
     llvm::Constant *Zero = llvm::Constant::getNullValue(HandleValue->getType());
     llvm::Value *NEZero = DtorBuilder.CreateICmpNE(HandleValue, Zero);
     DtorBuilder.CreateCondBr(NEZero, IfBlock, ExitBlock);
 
     DtorBuilder.SetInsertPoint(IfBlock);
     DtorBuilder.CreateCall(UnregisterFatbinFunc, HandleValue);
     DtorBuilder.CreateStore(Zero, GpuBinaryAddr);
     DtorBuilder.CreateBr(ExitBlock);
 
     DtorBuilder.SetInsertPoint(ExitBlock);
   } else {
     DtorBuilder.CreateCall(UnregisterFatbinFunc, HandleValue);
   }
   DtorBuilder.CreateRetVoid();
   return ModuleDtorFunc;
 }
 
 CGCUDARuntime *CodeGen::CreateNVCUDARuntime(CodeGenModule &CGM) {
   return new CGNVCUDARuntime(CGM);
 }
 
 void CGNVCUDARuntime::internalizeDeviceSideVar(
     const VarDecl *D, llvm::GlobalValue::LinkageTypes &Linkage) {
   // For -fno-gpu-rdc, host-side shadows of external declarations of device-side
   // global variables become internal definitions. These have to be internal in
   // order to prevent name conflicts with global host variables with the same
   // name in a different TUs.
   //
   // For -fgpu-rdc, the shadow variables should not be internalized because
   // they may be accessed by different TU.
   if (CGM.getLangOpts().GPURelocatableDeviceCode)
     return;
 
   // __shared__ variables are odd. Shadows do get created, but
   // they are not registered with the CUDA runtime, so they
   // can't really be used to access their device-side
   // counterparts. It's not clear yet whether it's nvcc's bug or
   // a feature, but we've got to do the same for compatibility.
   if (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() ||
       D->hasAttr<CUDASharedAttr>() ||
       D->getType()->isCUDADeviceBuiltinSurfaceType() ||
       D->getType()->isCUDADeviceBuiltinTextureType()) {
     Linkage = llvm::GlobalValue::InternalLinkage;
   }
 }
 
 void CGNVCUDARuntime::handleVarRegistration(const VarDecl *D,
                                             llvm::GlobalVariable &GV) {
   if (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>()) {
     // Shadow variables and their properties must be registered with CUDA
     // runtime. Skip Extern global variables, which will be registered in
     // the TU where they are defined.
     //
     // Don't register a C++17 inline variable. The local symbol can be
     // discarded and referencing a discarded local symbol from outside the
     // comdat (__cuda_register_globals) is disallowed by the ELF spec.
     //
     // HIP managed variables need to be always recorded in device and host
     // compilations for transformation.
     //
     // HIP managed variables and variables in CUDADeviceVarODRUsedByHost are
     // added to llvm.compiler-used, therefore they are safe to be registered.
     if ((!D->hasExternalStorage() && !D->isInline()) ||
         CGM.getContext().CUDADeviceVarODRUsedByHost.contains(D) ||
         D->hasAttr<HIPManagedAttr>()) {
       registerDeviceVar(D, GV, !D->hasDefinition(),
                         D->hasAttr<CUDAConstantAttr>());
     }
   } else if (D->getType()->isCUDADeviceBuiltinSurfaceType() ||
              D->getType()->isCUDADeviceBuiltinTextureType()) {
     // Builtin surfaces and textures and their template arguments are
     // also registered with CUDA runtime.
     const auto *TD = cast<ClassTemplateSpecializationDecl>(
         D->getType()->castAs<RecordType>()->getDecl());
     const TemplateArgumentList &Args = TD->getTemplateArgs();
     if (TD->hasAttr<CUDADeviceBuiltinSurfaceTypeAttr>()) {
       assert(Args.size() == 2 &&
              "Unexpected number of template arguments of CUDA device "
              "builtin surface type.");
       auto SurfType = Args[1].getAsIntegral();
       if (!D->hasExternalStorage())
         registerDeviceSurf(D, GV, !D->hasDefinition(), SurfType.getSExtValue());
     } else {
       assert(Args.size() == 3 &&
              "Unexpected number of template arguments of CUDA device "
              "builtin texture type.");
       auto TexType = Args[1].getAsIntegral();
       auto Normalized = Args[2].getAsIntegral();
       if (!D->hasExternalStorage())
         registerDeviceTex(D, GV, !D->hasDefinition(), TexType.getSExtValue(),
                           Normalized.getZExtValue());
     }
   }
 }
 
 // Transform managed variables to pointers to managed variables in device code.
 // Each use of the original managed variable is replaced by a load from the
 // transformed managed variable. The transformed managed variable contains
 // the address of managed memory which will be allocated by the runtime.
 void CGNVCUDARuntime::transformManagedVars() {
   for (auto &&Info : DeviceVars) {
     llvm::GlobalVariable *Var = Info.Var;
     if (Info.Flags.getKind() == DeviceVarFlags::Variable &&
         Info.Flags.isManaged()) {
       auto *ManagedVar = new llvm::GlobalVariable(
           CGM.getModule(), Var->getType(),
           /*isConstant=*/false, Var->getLinkage(),
           /*Init=*/Var->isDeclaration()
               ? nullptr
               : llvm::ConstantPointerNull::get(Var->getType()),
           /*Name=*/"", /*InsertBefore=*/nullptr,
           llvm::GlobalVariable::NotThreadLocal,
           CGM.getContext().getTargetAddressSpace(LangAS::cuda_device));
       ManagedVar->setDSOLocal(Var->isDSOLocal());
       ManagedVar->setVisibility(Var->getVisibility());
       ManagedVar->setExternallyInitialized(true);
       replaceManagedVar(Var, ManagedVar);
       ManagedVar->takeName(Var);
       Var->setName(Twine(ManagedVar->getName()) + ".managed");
       // Keep managed variables even if they are not used in device code since
       // they need to be allocated by the runtime.
       if (!Var->isDeclaration()) {
         assert(!ManagedVar->isDeclaration());
         CGM.addCompilerUsedGlobal(Var);
         CGM.addCompilerUsedGlobal(ManagedVar);
       }
     }
   }
 }
 
 // Returns module constructor to be added.
 llvm::Function *CGNVCUDARuntime::finalizeModule() {
   if (CGM.getLangOpts().CUDAIsDevice) {
     transformManagedVars();
 
     // Mark ODR-used device variables as compiler used to prevent it from being
     // eliminated by optimization. This is necessary for device variables
     // ODR-used by host functions. Sema correctly marks them as ODR-used no
     // matter whether they are ODR-used by device or host functions.
     //
     // We do not need to do this if the variable has used attribute since it
     // has already been added.
     //
     // Static device variables have been externalized at this point, therefore
     // variables with LLVM private or internal linkage need not be added.
     for (auto &&Info : DeviceVars) {
       auto Kind = Info.Flags.getKind();
       if (!Info.Var->isDeclaration() &&
           !llvm::GlobalValue::isLocalLinkage(Info.Var->getLinkage()) &&
           (Kind == DeviceVarFlags::Variable ||
            Kind == DeviceVarFlags::Surface ||
            Kind == DeviceVarFlags::Texture) &&
           Info.D->isUsed() && !Info.D->hasAttr<UsedAttr>()) {
         CGM.addCompilerUsedGlobal(Info.Var);
       }
     }
     return nullptr;
   }
   return makeModuleCtorFunction();
 }
 
 llvm::GlobalValue *CGNVCUDARuntime::getKernelHandle(llvm::Function *F,
                                                     GlobalDecl GD) {
   auto Loc = KernelHandles.find(F);
   if (Loc != KernelHandles.end())
     return Loc->second;
 
   if (!CGM.getLangOpts().HIP) {
     KernelHandles[F] = F;
     KernelStubs[F] = F;
     return F;
   }
 
   auto *Var = new llvm::GlobalVariable(
       TheModule, F->getType(), /*isConstant=*/true, F->getLinkage(),
       /*Initializer=*/nullptr,
       CGM.getMangledName(
           GD.getWithKernelReferenceKind(KernelReferenceKind::Kernel)));
   Var->setAlignment(CGM.getPointerAlign().getAsAlign());
   Var->setDSOLocal(F->isDSOLocal());
   Var->setVisibility(F->getVisibility());
   CGM.maybeSetTrivialComdat(*GD.getDecl(), *Var);
   KernelHandles[F] = Var;
   KernelStubs[Var] = F;
   return Var;
 }
diff --git a/contrib/llvm-project/clang/lib/CodeGen/CGExprCXX.cpp b/contrib/llvm-project/clang/lib/CodeGen/CGExprCXX.cpp
index f06d21861740..3a3acf98f309 100644
--- a/contrib/llvm-project/clang/lib/CodeGen/CGExprCXX.cpp
+++ b/contrib/llvm-project/clang/lib/CodeGen/CGExprCXX.cpp
@@ -1,2319 +1,2319 @@
 //===--- CGExprCXX.cpp - Emit LLVM Code for C++ expressions ---------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This contains code dealing with code generation of C++ expressions
 //
 //===----------------------------------------------------------------------===//
 
 #include "CGCUDARuntime.h"
 #include "CGCXXABI.h"
 #include "CGDebugInfo.h"
 #include "CGObjCRuntime.h"
 #include "CodeGenFunction.h"
 #include "ConstantEmitter.h"
 #include "TargetInfo.h"
 #include "clang/Basic/CodeGenOptions.h"
 #include "clang/CodeGen/CGFunctionInfo.h"
 #include "llvm/IR/Intrinsics.h"
 
 using namespace clang;
 using namespace CodeGen;
 
 namespace {
 struct MemberCallInfo {
   RequiredArgs ReqArgs;
   // Number of prefix arguments for the call. Ignores the `this` pointer.
   unsigned PrefixSize;
 };
 }
 
 static MemberCallInfo
 commonEmitCXXMemberOrOperatorCall(CodeGenFunction &CGF, const CXXMethodDecl *MD,
                                   llvm::Value *This, llvm::Value *ImplicitParam,
                                   QualType ImplicitParamTy, const CallExpr *CE,
                                   CallArgList &Args, CallArgList *RtlArgs) {
   assert(CE == nullptr || isa<CXXMemberCallExpr>(CE) ||
          isa<CXXOperatorCallExpr>(CE));
   assert(MD->isInstance() &&
          "Trying to emit a member or operator call expr on a static method!");
 
   // Push the this ptr.
   const CXXRecordDecl *RD =
       CGF.CGM.getCXXABI().getThisArgumentTypeForMethod(MD);
   Args.add(RValue::get(This), CGF.getTypes().DeriveThisType(RD, MD));
 
   // If there is an implicit parameter (e.g. VTT), emit it.
   if (ImplicitParam) {
     Args.add(RValue::get(ImplicitParam), ImplicitParamTy);
   }
 
   const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
   RequiredArgs required = RequiredArgs::forPrototypePlus(FPT, Args.size());
   unsigned PrefixSize = Args.size() - 1;
 
   // And the rest of the call args.
   if (RtlArgs) {
     // Special case: if the caller emitted the arguments right-to-left already
     // (prior to emitting the *this argument), we're done. This happens for
     // assignment operators.
     Args.addFrom(*RtlArgs);
   } else if (CE) {
     // Special case: skip first argument of CXXOperatorCall (it is "this").
     unsigned ArgsToSkip = isa<CXXOperatorCallExpr>(CE) ? 1 : 0;
     CGF.EmitCallArgs(Args, FPT, drop_begin(CE->arguments(), ArgsToSkip),
                      CE->getDirectCallee());
   } else {
     assert(
         FPT->getNumParams() == 0 &&
         "No CallExpr specified for function with non-zero number of arguments");
   }
   return {required, PrefixSize};
 }
 
 RValue CodeGenFunction::EmitCXXMemberOrOperatorCall(
     const CXXMethodDecl *MD, const CGCallee &Callee,
     ReturnValueSlot ReturnValue,
     llvm::Value *This, llvm::Value *ImplicitParam, QualType ImplicitParamTy,
     const CallExpr *CE, CallArgList *RtlArgs) {
   const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
   CallArgList Args;
   MemberCallInfo CallInfo = commonEmitCXXMemberOrOperatorCall(
       *this, MD, This, ImplicitParam, ImplicitParamTy, CE, Args, RtlArgs);
   auto &FnInfo = CGM.getTypes().arrangeCXXMethodCall(
       Args, FPT, CallInfo.ReqArgs, CallInfo.PrefixSize);
   return EmitCall(FnInfo, Callee, ReturnValue, Args, nullptr,
                   CE && CE == MustTailCall,
                   CE ? CE->getExprLoc() : SourceLocation());
 }
 
 RValue CodeGenFunction::EmitCXXDestructorCall(
     GlobalDecl Dtor, const CGCallee &Callee, llvm::Value *This, QualType ThisTy,
     llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *CE) {
   const CXXMethodDecl *DtorDecl = cast<CXXMethodDecl>(Dtor.getDecl());
 
   assert(!ThisTy.isNull());
   assert(ThisTy->getAsCXXRecordDecl() == DtorDecl->getParent() &&
          "Pointer/Object mixup");
 
   LangAS SrcAS = ThisTy.getAddressSpace();
   LangAS DstAS = DtorDecl->getMethodQualifiers().getAddressSpace();
   if (SrcAS != DstAS) {
     QualType DstTy = DtorDecl->getThisType();
     llvm::Type *NewType = CGM.getTypes().ConvertType(DstTy);
     This = getTargetHooks().performAddrSpaceCast(*this, This, SrcAS, DstAS,
                                                  NewType);
   }
 
   CallArgList Args;
   commonEmitCXXMemberOrOperatorCall(*this, DtorDecl, This, ImplicitParam,
                                     ImplicitParamTy, CE, Args, nullptr);
   return EmitCall(CGM.getTypes().arrangeCXXStructorDeclaration(Dtor), Callee,
                   ReturnValueSlot(), Args, nullptr, CE && CE == MustTailCall,
                   CE ? CE->getExprLoc() : SourceLocation{});
 }
 
 RValue CodeGenFunction::EmitCXXPseudoDestructorExpr(
                                             const CXXPseudoDestructorExpr *E) {
   QualType DestroyedType = E->getDestroyedType();
   if (DestroyedType.hasStrongOrWeakObjCLifetime()) {
     // Automatic Reference Counting:
     //   If the pseudo-expression names a retainable object with weak or
     //   strong lifetime, the object shall be released.
     Expr *BaseExpr = E->getBase();
     Address BaseValue = Address::invalid();
     Qualifiers BaseQuals;
 
     // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
     if (E->isArrow()) {
       BaseValue = EmitPointerWithAlignment(BaseExpr);
       const auto *PTy = BaseExpr->getType()->castAs<PointerType>();
       BaseQuals = PTy->getPointeeType().getQualifiers();
     } else {
       LValue BaseLV = EmitLValue(BaseExpr);
       BaseValue = BaseLV.getAddress(*this);
       QualType BaseTy = BaseExpr->getType();
       BaseQuals = BaseTy.getQualifiers();
     }
 
     switch (DestroyedType.getObjCLifetime()) {
     case Qualifiers::OCL_None:
     case Qualifiers::OCL_ExplicitNone:
     case Qualifiers::OCL_Autoreleasing:
       break;
 
     case Qualifiers::OCL_Strong:
       EmitARCRelease(Builder.CreateLoad(BaseValue,
                         DestroyedType.isVolatileQualified()),
                      ARCPreciseLifetime);
       break;
 
     case Qualifiers::OCL_Weak:
       EmitARCDestroyWeak(BaseValue);
       break;
     }
   } else {
     // C++ [expr.pseudo]p1:
     //   The result shall only be used as the operand for the function call
     //   operator (), and the result of such a call has type void. The only
     //   effect is the evaluation of the postfix-expression before the dot or
     //   arrow.
     EmitIgnoredExpr(E->getBase());
   }
 
   return RValue::get(nullptr);
 }
 
 static CXXRecordDecl *getCXXRecord(const Expr *E) {
   QualType T = E->getType();
   if (const PointerType *PTy = T->getAs<PointerType>())
     T = PTy->getPointeeType();
   const RecordType *Ty = T->castAs<RecordType>();
   return cast<CXXRecordDecl>(Ty->getDecl());
 }
 
 // Note: This function also emit constructor calls to support a MSVC
 // extensions allowing explicit constructor function call.
 RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
                                               ReturnValueSlot ReturnValue) {
   const Expr *callee = CE->getCallee()->IgnoreParens();
 
   if (isa<BinaryOperator>(callee))
     return EmitCXXMemberPointerCallExpr(CE, ReturnValue);
 
   const MemberExpr *ME = cast<MemberExpr>(callee);
   const CXXMethodDecl *MD = cast<CXXMethodDecl>(ME->getMemberDecl());
 
   if (MD->isStatic()) {
     // The method is static, emit it as we would a regular call.
     CGCallee callee =
         CGCallee::forDirect(CGM.GetAddrOfFunction(MD), GlobalDecl(MD));
     return EmitCall(getContext().getPointerType(MD->getType()), callee, CE,
                     ReturnValue);
   }
 
   bool HasQualifier = ME->hasQualifier();
   NestedNameSpecifier *Qualifier = HasQualifier ? ME->getQualifier() : nullptr;
   bool IsArrow = ME->isArrow();
   const Expr *Base = ME->getBase();
 
   return EmitCXXMemberOrOperatorMemberCallExpr(
       CE, MD, ReturnValue, HasQualifier, Qualifier, IsArrow, Base);
 }
 
 RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr(
     const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,
     bool HasQualifier, NestedNameSpecifier *Qualifier, bool IsArrow,
     const Expr *Base) {
   assert(isa<CXXMemberCallExpr>(CE) || isa<CXXOperatorCallExpr>(CE));
 
   // Compute the object pointer.
   bool CanUseVirtualCall = MD->isVirtual() && !HasQualifier;
 
   const CXXMethodDecl *DevirtualizedMethod = nullptr;
   if (CanUseVirtualCall &&
       MD->getDevirtualizedMethod(Base, getLangOpts().AppleKext)) {
     const CXXRecordDecl *BestDynamicDecl = Base->getBestDynamicClassType();
     DevirtualizedMethod = MD->getCorrespondingMethodInClass(BestDynamicDecl);
     assert(DevirtualizedMethod);
     const CXXRecordDecl *DevirtualizedClass = DevirtualizedMethod->getParent();
     const Expr *Inner = Base->IgnoreParenBaseCasts();
     if (DevirtualizedMethod->getReturnType().getCanonicalType() !=
         MD->getReturnType().getCanonicalType())
       // If the return types are not the same, this might be a case where more
       // code needs to run to compensate for it. For example, the derived
       // method might return a type that inherits form from the return
       // type of MD and has a prefix.
       // For now we just avoid devirtualizing these covariant cases.
       DevirtualizedMethod = nullptr;
     else if (getCXXRecord(Inner) == DevirtualizedClass)
       // If the class of the Inner expression is where the dynamic method
       // is defined, build the this pointer from it.
       Base = Inner;
     else if (getCXXRecord(Base) != DevirtualizedClass) {
       // If the method is defined in a class that is not the best dynamic
       // one or the one of the full expression, we would have to build
       // a derived-to-base cast to compute the correct this pointer, but
       // we don't have support for that yet, so do a virtual call.
       DevirtualizedMethod = nullptr;
     }
   }
 
   bool TrivialForCodegen =
       MD->isTrivial() || (MD->isDefaulted() && MD->getParent()->isUnion());
   bool TrivialAssignment =
       TrivialForCodegen &&
       (MD->isCopyAssignmentOperator() || MD->isMoveAssignmentOperator()) &&
       !MD->getParent()->mayInsertExtraPadding();
 
   // C++17 demands that we evaluate the RHS of a (possibly-compound) assignment
   // operator before the LHS.
   CallArgList RtlArgStorage;
   CallArgList *RtlArgs = nullptr;
   LValue TrivialAssignmentRHS;
   if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(CE)) {
     if (OCE->isAssignmentOp()) {
       if (TrivialAssignment) {
         TrivialAssignmentRHS = EmitLValue(CE->getArg(1));
       } else {
         RtlArgs = &RtlArgStorage;
         EmitCallArgs(*RtlArgs, MD->getType()->castAs<FunctionProtoType>(),
                      drop_begin(CE->arguments(), 1), CE->getDirectCallee(),
                      /*ParamsToSkip*/0, EvaluationOrder::ForceRightToLeft);
       }
     }
   }
 
   LValue This;
   if (IsArrow) {
     LValueBaseInfo BaseInfo;
     TBAAAccessInfo TBAAInfo;
     Address ThisValue = EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo);
     This = MakeAddrLValue(ThisValue, Base->getType(), BaseInfo, TBAAInfo);
   } else {
     This = EmitLValue(Base);
   }
 
   if (const CXXConstructorDecl *Ctor = dyn_cast<CXXConstructorDecl>(MD)) {
     // This is the MSVC p->Ctor::Ctor(...) extension. We assume that's
     // constructing a new complete object of type Ctor.
     assert(!RtlArgs);
     assert(ReturnValue.isNull() && "Constructor shouldn't have return value");
     CallArgList Args;
     commonEmitCXXMemberOrOperatorCall(
         *this, Ctor, This.getPointer(*this), /*ImplicitParam=*/nullptr,
         /*ImplicitParamTy=*/QualType(), CE, Args, nullptr);
 
     EmitCXXConstructorCall(Ctor, Ctor_Complete, /*ForVirtualBase=*/false,
                            /*Delegating=*/false, This.getAddress(*this), Args,
                            AggValueSlot::DoesNotOverlap, CE->getExprLoc(),
                            /*NewPointerIsChecked=*/false);
     return RValue::get(nullptr);
   }
 
   if (TrivialForCodegen) {
     if (isa<CXXDestructorDecl>(MD))
       return RValue::get(nullptr);
 
     if (TrivialAssignment) {
       // We don't like to generate the trivial copy/move assignment operator
       // when it isn't necessary; just produce the proper effect here.
       // It's important that we use the result of EmitLValue here rather than
       // emitting call arguments, in order to preserve TBAA information from
       // the RHS.
       LValue RHS = isa<CXXOperatorCallExpr>(CE)
                        ? TrivialAssignmentRHS
                        : EmitLValue(*CE->arg_begin());
       EmitAggregateAssign(This, RHS, CE->getType());
       return RValue::get(This.getPointer(*this));
     }
 
     assert(MD->getParent()->mayInsertExtraPadding() &&
            "unknown trivial member function");
   }
 
   // Compute the function type we're calling.
   const CXXMethodDecl *CalleeDecl =
       DevirtualizedMethod ? DevirtualizedMethod : MD;
   const CGFunctionInfo *FInfo = nullptr;
   if (const auto *Dtor = dyn_cast<CXXDestructorDecl>(CalleeDecl))
     FInfo = &CGM.getTypes().arrangeCXXStructorDeclaration(
         GlobalDecl(Dtor, Dtor_Complete));
   else
     FInfo = &CGM.getTypes().arrangeCXXMethodDeclaration(CalleeDecl);
 
   llvm::FunctionType *Ty = CGM.getTypes().GetFunctionType(*FInfo);
 
   // C++11 [class.mfct.non-static]p2:
   //   If a non-static member function of a class X is called for an object that
   //   is not of type X, or of a type derived from X, the behavior is undefined.
   SourceLocation CallLoc;
   ASTContext &C = getContext();
   if (CE)
     CallLoc = CE->getExprLoc();
 
   SanitizerSet SkippedChecks;
   if (const auto *CMCE = dyn_cast<CXXMemberCallExpr>(CE)) {
     auto *IOA = CMCE->getImplicitObjectArgument();
     bool IsImplicitObjectCXXThis = IsWrappedCXXThis(IOA);
     if (IsImplicitObjectCXXThis)
       SkippedChecks.set(SanitizerKind::Alignment, true);
     if (IsImplicitObjectCXXThis || isa<DeclRefExpr>(IOA))
       SkippedChecks.set(SanitizerKind::Null, true);
   }
   EmitTypeCheck(CodeGenFunction::TCK_MemberCall, CallLoc,
                 This.getPointer(*this),
                 C.getRecordType(CalleeDecl->getParent()),
                 /*Alignment=*/CharUnits::Zero(), SkippedChecks);
 
   // C++ [class.virtual]p12:
   //   Explicit qualification with the scope operator (5.1) suppresses the
   //   virtual call mechanism.
   //
   // We also don't emit a virtual call if the base expression has a record type
   // because then we know what the type is.
   bool UseVirtualCall = CanUseVirtualCall && !DevirtualizedMethod;
 
   if (const CXXDestructorDecl *Dtor = dyn_cast<CXXDestructorDecl>(CalleeDecl)) {
     assert(CE->arg_begin() == CE->arg_end() &&
            "Destructor shouldn't have explicit parameters");
     assert(ReturnValue.isNull() && "Destructor shouldn't have return value");
     if (UseVirtualCall) {
       CGM.getCXXABI().EmitVirtualDestructorCall(*this, Dtor, Dtor_Complete,
                                                 This.getAddress(*this),
                                                 cast<CXXMemberCallExpr>(CE));
     } else {
       GlobalDecl GD(Dtor, Dtor_Complete);
       CGCallee Callee;
       if (getLangOpts().AppleKext && Dtor->isVirtual() && HasQualifier)
         Callee = BuildAppleKextVirtualCall(Dtor, Qualifier, Ty);
       else if (!DevirtualizedMethod)
         Callee =
             CGCallee::forDirect(CGM.getAddrOfCXXStructor(GD, FInfo, Ty), GD);
       else {
         Callee = CGCallee::forDirect(CGM.GetAddrOfFunction(GD, Ty), GD);
       }
 
       QualType ThisTy =
           IsArrow ? Base->getType()->getPointeeType() : Base->getType();
       EmitCXXDestructorCall(GD, Callee, This.getPointer(*this), ThisTy,
                             /*ImplicitParam=*/nullptr,
                             /*ImplicitParamTy=*/QualType(), CE);
     }
     return RValue::get(nullptr);
   }
 
   // FIXME: Uses of 'MD' past this point need to be audited. We may need to use
   // 'CalleeDecl' instead.
 
   CGCallee Callee;
   if (UseVirtualCall) {
     Callee = CGCallee::forVirtual(CE, MD, This.getAddress(*this), Ty);
   } else {
     if (SanOpts.has(SanitizerKind::CFINVCall) &&
         MD->getParent()->isDynamicClass()) {
       llvm::Value *VTable;
       const CXXRecordDecl *RD;
       std::tie(VTable, RD) = CGM.getCXXABI().LoadVTablePtr(
           *this, This.getAddress(*this), CalleeDecl->getParent());
       EmitVTablePtrCheckForCall(RD, VTable, CFITCK_NVCall, CE->getBeginLoc());
     }
 
     if (getLangOpts().AppleKext && MD->isVirtual() && HasQualifier)
       Callee = BuildAppleKextVirtualCall(MD, Qualifier, Ty);
     else if (!DevirtualizedMethod)
       Callee =
           CGCallee::forDirect(CGM.GetAddrOfFunction(MD, Ty), GlobalDecl(MD));
     else {
       Callee =
           CGCallee::forDirect(CGM.GetAddrOfFunction(DevirtualizedMethod, Ty),
                               GlobalDecl(DevirtualizedMethod));
     }
   }
 
   if (MD->isVirtual()) {
     Address NewThisAddr =
         CGM.getCXXABI().adjustThisArgumentForVirtualFunctionCall(
             *this, CalleeDecl, This.getAddress(*this), UseVirtualCall);
     This.setAddress(NewThisAddr);
   }
 
   return EmitCXXMemberOrOperatorCall(
       CalleeDecl, Callee, ReturnValue, This.getPointer(*this),
       /*ImplicitParam=*/nullptr, QualType(), CE, RtlArgs);
 }
 
 RValue
 CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
                                               ReturnValueSlot ReturnValue) {
   const BinaryOperator *BO =
       cast<BinaryOperator>(E->getCallee()->IgnoreParens());
   const Expr *BaseExpr = BO->getLHS();
   const Expr *MemFnExpr = BO->getRHS();
 
   const auto *MPT = MemFnExpr->getType()->castAs<MemberPointerType>();
   const auto *FPT = MPT->getPointeeType()->castAs<FunctionProtoType>();
   const auto *RD =
       cast<CXXRecordDecl>(MPT->getClass()->castAs<RecordType>()->getDecl());
 
   // Emit the 'this' pointer.
   Address This = Address::invalid();
   if (BO->getOpcode() == BO_PtrMemI)
     This = EmitPointerWithAlignment(BaseExpr);
   else
     This = EmitLValue(BaseExpr).getAddress(*this);
 
   EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.getPointer(),
                 QualType(MPT->getClass(), 0));
 
   // Get the member function pointer.
   llvm::Value *MemFnPtr = EmitScalarExpr(MemFnExpr);
 
   // Ask the ABI to load the callee.  Note that This is modified.
   llvm::Value *ThisPtrForCall = nullptr;
   CGCallee Callee =
     CGM.getCXXABI().EmitLoadOfMemberFunctionPointer(*this, BO, This,
                                              ThisPtrForCall, MemFnPtr, MPT);
 
   CallArgList Args;
 
   QualType ThisType =
     getContext().getPointerType(getContext().getTagDeclType(RD));
 
   // Push the this ptr.
   Args.add(RValue::get(ThisPtrForCall), ThisType);
 
   RequiredArgs required = RequiredArgs::forPrototypePlus(FPT, 1);
 
   // And the rest of the call args
   EmitCallArgs(Args, FPT, E->arguments());
   return EmitCall(CGM.getTypes().arrangeCXXMethodCall(Args, FPT, required,
                                                       /*PrefixSize=*/0),
                   Callee, ReturnValue, Args, nullptr, E == MustTailCall,
                   E->getExprLoc());
 }
 
 RValue
 CodeGenFunction::EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
                                                const CXXMethodDecl *MD,
                                                ReturnValueSlot ReturnValue) {
   assert(MD->isInstance() &&
          "Trying to emit a member call expr on a static method!");
   return EmitCXXMemberOrOperatorMemberCallExpr(
       E, MD, ReturnValue, /*HasQualifier=*/false, /*Qualifier=*/nullptr,
       /*IsArrow=*/false, E->getArg(0));
 }
 
 RValue CodeGenFunction::EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
                                                ReturnValueSlot ReturnValue) {
   return CGM.getCUDARuntime().EmitCUDAKernelCallExpr(*this, E, ReturnValue);
 }
 
 static void EmitNullBaseClassInitialization(CodeGenFunction &CGF,
                                             Address DestPtr,
                                             const CXXRecordDecl *Base) {
   if (Base->isEmpty())
     return;
 
   DestPtr = CGF.Builder.CreateElementBitCast(DestPtr, CGF.Int8Ty);
 
   const ASTRecordLayout &Layout = CGF.getContext().getASTRecordLayout(Base);
   CharUnits NVSize = Layout.getNonVirtualSize();
 
   // We cannot simply zero-initialize the entire base sub-object if vbptrs are
   // present, they are initialized by the most derived class before calling the
   // constructor.
   SmallVector<std::pair<CharUnits, CharUnits>, 1> Stores;
   Stores.emplace_back(CharUnits::Zero(), NVSize);
 
   // Each store is split by the existence of a vbptr.
   CharUnits VBPtrWidth = CGF.getPointerSize();
   std::vector<CharUnits> VBPtrOffsets =
       CGF.CGM.getCXXABI().getVBPtrOffsets(Base);
   for (CharUnits VBPtrOffset : VBPtrOffsets) {
     // Stop before we hit any virtual base pointers located in virtual bases.
     if (VBPtrOffset >= NVSize)
       break;
     std::pair<CharUnits, CharUnits> LastStore = Stores.pop_back_val();
     CharUnits LastStoreOffset = LastStore.first;
     CharUnits LastStoreSize = LastStore.second;
 
     CharUnits SplitBeforeOffset = LastStoreOffset;
     CharUnits SplitBeforeSize = VBPtrOffset - SplitBeforeOffset;
     assert(!SplitBeforeSize.isNegative() && "negative store size!");
     if (!SplitBeforeSize.isZero())
       Stores.emplace_back(SplitBeforeOffset, SplitBeforeSize);
 
     CharUnits SplitAfterOffset = VBPtrOffset + VBPtrWidth;
     CharUnits SplitAfterSize = LastStoreSize - SplitAfterOffset;
     assert(!SplitAfterSize.isNegative() && "negative store size!");
     if (!SplitAfterSize.isZero())
       Stores.emplace_back(SplitAfterOffset, SplitAfterSize);
   }
 
   // If the type contains a pointer to data member we can't memset it to zero.
   // Instead, create a null constant and copy it to the destination.
   // TODO: there are other patterns besides zero that we can usefully memset,
   // like -1, which happens to be the pattern used by member-pointers.
   // TODO: isZeroInitializable can be over-conservative in the case where a
   // virtual base contains a member pointer.
   llvm::Constant *NullConstantForBase = CGF.CGM.EmitNullConstantForBase(Base);
   if (!NullConstantForBase->isNullValue()) {
     llvm::GlobalVariable *NullVariable = new llvm::GlobalVariable(
         CGF.CGM.getModule(), NullConstantForBase->getType(),
         /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage,
         NullConstantForBase, Twine());
 
     CharUnits Align = std::max(Layout.getNonVirtualAlignment(),
                                DestPtr.getAlignment());
     NullVariable->setAlignment(Align.getAsAlign());
 
     Address SrcPtr = Address(CGF.EmitCastToVoidPtr(NullVariable), Align);
 
     // Get and call the appropriate llvm.memcpy overload.
     for (std::pair<CharUnits, CharUnits> Store : Stores) {
       CharUnits StoreOffset = Store.first;
       CharUnits StoreSize = Store.second;
       llvm::Value *StoreSizeVal = CGF.CGM.getSize(StoreSize);
       CGF.Builder.CreateMemCpy(
           CGF.Builder.CreateConstInBoundsByteGEP(DestPtr, StoreOffset),
           CGF.Builder.CreateConstInBoundsByteGEP(SrcPtr, StoreOffset),
           StoreSizeVal);
     }
 
   // Otherwise, just memset the whole thing to zero.  This is legal
   // because in LLVM, all default initializers (other than the ones we just
   // handled above) are guaranteed to have a bit pattern of all zeros.
   } else {
     for (std::pair<CharUnits, CharUnits> Store : Stores) {
       CharUnits StoreOffset = Store.first;
       CharUnits StoreSize = Store.second;
       llvm::Value *StoreSizeVal = CGF.CGM.getSize(StoreSize);
       CGF.Builder.CreateMemSet(
           CGF.Builder.CreateConstInBoundsByteGEP(DestPtr, StoreOffset),
           CGF.Builder.getInt8(0), StoreSizeVal);
     }
   }
 }
 
 void
 CodeGenFunction::EmitCXXConstructExpr(const CXXConstructExpr *E,
                                       AggValueSlot Dest) {
   assert(!Dest.isIgnored() && "Must have a destination!");
   const CXXConstructorDecl *CD = E->getConstructor();
 
   // If we require zero initialization before (or instead of) calling the
   // constructor, as can be the case with a non-user-provided default
   // constructor, emit the zero initialization now, unless destination is
   // already zeroed.
   if (E->requiresZeroInitialization() && !Dest.isZeroed()) {
     switch (E->getConstructionKind()) {
     case CXXConstructExpr::CK_Delegating:
     case CXXConstructExpr::CK_Complete:
       EmitNullInitialization(Dest.getAddress(), E->getType());
       break;
     case CXXConstructExpr::CK_VirtualBase:
     case CXXConstructExpr::CK_NonVirtualBase:
       EmitNullBaseClassInitialization(*this, Dest.getAddress(),
                                       CD->getParent());
       break;
     }
   }
 
   // If this is a call to a trivial default constructor, do nothing.
   if (CD->isTrivial() && CD->isDefaultConstructor())
     return;
 
   // Elide the constructor if we're constructing from a temporary.
   if (getLangOpts().ElideConstructors && E->isElidable()) {
     // FIXME: This only handles the simplest case, where the source object
     //        is passed directly as the first argument to the constructor.
     //        This should also handle stepping though implicit casts and
     //        conversion sequences which involve two steps, with a
     //        conversion operator followed by a converting constructor.
     const Expr *SrcObj = E->getArg(0);
     assert(SrcObj->isTemporaryObject(getContext(), CD->getParent()));
     assert(
         getContext().hasSameUnqualifiedType(E->getType(), SrcObj->getType()));
     EmitAggExpr(SrcObj, Dest);
     return;
   }
 
   if (const ArrayType *arrayType
         = getContext().getAsArrayType(E->getType())) {
     EmitCXXAggrConstructorCall(CD, arrayType, Dest.getAddress(), E,
                                Dest.isSanitizerChecked());
   } else {
     CXXCtorType Type = Ctor_Complete;
     bool ForVirtualBase = false;
     bool Delegating = false;
 
     switch (E->getConstructionKind()) {
      case CXXConstructExpr::CK_Delegating:
       // We should be emitting a constructor; GlobalDecl will assert this
       Type = CurGD.getCtorType();
       Delegating = true;
       break;
 
      case CXXConstructExpr::CK_Complete:
       Type = Ctor_Complete;
       break;
 
      case CXXConstructExpr::CK_VirtualBase:
       ForVirtualBase = true;
       LLVM_FALLTHROUGH;
 
      case CXXConstructExpr::CK_NonVirtualBase:
       Type = Ctor_Base;
      }
 
      // Call the constructor.
      EmitCXXConstructorCall(CD, Type, ForVirtualBase, Delegating, Dest, E);
   }
 }
 
 void CodeGenFunction::EmitSynthesizedCXXCopyCtor(Address Dest, Address Src,
                                                  const Expr *Exp) {
   if (const ExprWithCleanups *E = dyn_cast<ExprWithCleanups>(Exp))
     Exp = E->getSubExpr();
   assert(isa<CXXConstructExpr>(Exp) &&
          "EmitSynthesizedCXXCopyCtor - unknown copy ctor expr");
   const CXXConstructExpr* E = cast<CXXConstructExpr>(Exp);
   const CXXConstructorDecl *CD = E->getConstructor();
   RunCleanupsScope Scope(*this);
 
   // If we require zero initialization before (or instead of) calling the
   // constructor, as can be the case with a non-user-provided default
   // constructor, emit the zero initialization now.
   // FIXME. Do I still need this for a copy ctor synthesis?
   if (E->requiresZeroInitialization())
     EmitNullInitialization(Dest, E->getType());
 
   assert(!getContext().getAsConstantArrayType(E->getType())
          && "EmitSynthesizedCXXCopyCtor - Copied-in Array");
   EmitSynthesizedCXXCopyCtorCall(CD, Dest, Src, E);
 }
 
 static CharUnits CalculateCookiePadding(CodeGenFunction &CGF,
                                         const CXXNewExpr *E) {
   if (!E->isArray())
     return CharUnits::Zero();
 
   // No cookie is required if the operator new[] being used is the
   // reserved placement operator new[].
   if (E->getOperatorNew()->isReservedGlobalPlacementOperator())
     return CharUnits::Zero();
 
   return CGF.CGM.getCXXABI().GetArrayCookieSize(E);
 }
 
 static llvm::Value *EmitCXXNewAllocSize(CodeGenFunction &CGF,
                                         const CXXNewExpr *e,
                                         unsigned minElements,
                                         llvm::Value *&numElements,
                                         llvm::Value *&sizeWithoutCookie) {
   QualType type = e->getAllocatedType();
 
   if (!e->isArray()) {
     CharUnits typeSize = CGF.getContext().getTypeSizeInChars(type);
     sizeWithoutCookie
       = llvm::ConstantInt::get(CGF.SizeTy, typeSize.getQuantity());
     return sizeWithoutCookie;
   }
 
   // The width of size_t.
   unsigned sizeWidth = CGF.SizeTy->getBitWidth();
 
   // Figure out the cookie size.
   llvm::APInt cookieSize(sizeWidth,
                          CalculateCookiePadding(CGF, e).getQuantity());
 
   // Emit the array size expression.
   // We multiply the size of all dimensions for NumElements.
   // e.g for 'int[2][3]', ElemType is 'int' and NumElements is 6.
   numElements =
     ConstantEmitter(CGF).tryEmitAbstract(*e->getArraySize(), e->getType());
   if (!numElements)
     numElements = CGF.EmitScalarExpr(*e->getArraySize());
   assert(isa<llvm::IntegerType>(numElements->getType()));
 
   // The number of elements can be have an arbitrary integer type;
   // essentially, we need to multiply it by a constant factor, add a
   // cookie size, and verify that the result is representable as a
   // size_t.  That's just a gloss, though, and it's wrong in one
   // important way: if the count is negative, it's an error even if
   // the cookie size would bring the total size >= 0.
   bool isSigned
     = (*e->getArraySize())->getType()->isSignedIntegerOrEnumerationType();
   llvm::IntegerType *numElementsType
     = cast<llvm::IntegerType>(numElements->getType());
   unsigned numElementsWidth = numElementsType->getBitWidth();
 
   // Compute the constant factor.
   llvm::APInt arraySizeMultiplier(sizeWidth, 1);
   while (const ConstantArrayType *CAT
              = CGF.getContext().getAsConstantArrayType(type)) {
     type = CAT->getElementType();
     arraySizeMultiplier *= CAT->getSize();
   }
 
   CharUnits typeSize = CGF.getContext().getTypeSizeInChars(type);
   llvm::APInt typeSizeMultiplier(sizeWidth, typeSize.getQuantity());
   typeSizeMultiplier *= arraySizeMultiplier;
 
   // This will be a size_t.
   llvm::Value *size;
 
   // If someone is doing 'new int[42]' there is no need to do a dynamic check.
   // Don't bloat the -O0 code.
   if (llvm::ConstantInt *numElementsC =
         dyn_cast<llvm::ConstantInt>(numElements)) {
     const llvm::APInt &count = numElementsC->getValue();
 
     bool hasAnyOverflow = false;
 
     // If 'count' was a negative number, it's an overflow.
     if (isSigned && count.isNegative())
       hasAnyOverflow = true;
 
     // We want to do all this arithmetic in size_t.  If numElements is
     // wider than that, check whether it's already too big, and if so,
     // overflow.
     else if (numElementsWidth > sizeWidth &&
              numElementsWidth - sizeWidth > count.countLeadingZeros())
       hasAnyOverflow = true;
 
     // Okay, compute a count at the right width.
     llvm::APInt adjustedCount = count.zextOrTrunc(sizeWidth);
 
     // If there is a brace-initializer, we cannot allocate fewer elements than
     // there are initializers. If we do, that's treated like an overflow.
     if (adjustedCount.ult(minElements))
       hasAnyOverflow = true;
 
     // Scale numElements by that.  This might overflow, but we don't
     // care because it only overflows if allocationSize does, too, and
     // if that overflows then we shouldn't use this.
     numElements = llvm::ConstantInt::get(CGF.SizeTy,
                                          adjustedCount * arraySizeMultiplier);
 
     // Compute the size before cookie, and track whether it overflowed.
     bool overflow;
     llvm::APInt allocationSize
       = adjustedCount.umul_ov(typeSizeMultiplier, overflow);
     hasAnyOverflow |= overflow;
 
     // Add in the cookie, and check whether it's overflowed.
     if (cookieSize != 0) {
       // Save the current size without a cookie.  This shouldn't be
       // used if there was overflow.
       sizeWithoutCookie = llvm::ConstantInt::get(CGF.SizeTy, allocationSize);
 
       allocationSize = allocationSize.uadd_ov(cookieSize, overflow);
       hasAnyOverflow |= overflow;
     }
 
     // On overflow, produce a -1 so operator new will fail.
     if (hasAnyOverflow) {
       size = llvm::Constant::getAllOnesValue(CGF.SizeTy);
     } else {
       size = llvm::ConstantInt::get(CGF.SizeTy, allocationSize);
     }
 
   // Otherwise, we might need to use the overflow intrinsics.
   } else {
     // There are up to five conditions we need to test for:
     // 1) if isSigned, we need to check whether numElements is negative;
     // 2) if numElementsWidth > sizeWidth, we need to check whether
     //   numElements is larger than something representable in size_t;
     // 3) if minElements > 0, we need to check whether numElements is smaller
     //    than that.
     // 4) we need to compute
     //      sizeWithoutCookie := numElements * typeSizeMultiplier
     //    and check whether it overflows; and
     // 5) if we need a cookie, we need to compute
     //      size := sizeWithoutCookie + cookieSize
     //    and check whether it overflows.
 
     llvm::Value *hasOverflow = nullptr;
 
     // If numElementsWidth > sizeWidth, then one way or another, we're
     // going to have to do a comparison for (2), and this happens to
     // take care of (1), too.
     if (numElementsWidth > sizeWidth) {
       llvm::APInt threshold(numElementsWidth, 1);
       threshold <<= sizeWidth;
 
       llvm::Value *thresholdV
         = llvm::ConstantInt::get(numElementsType, threshold);
 
       hasOverflow = CGF.Builder.CreateICmpUGE(numElements, thresholdV);
       numElements = CGF.Builder.CreateTrunc(numElements, CGF.SizeTy);
 
     // Otherwise, if we're signed, we want to sext up to size_t.
     } else if (isSigned) {
       if (numElementsWidth < sizeWidth)
         numElements = CGF.Builder.CreateSExt(numElements, CGF.SizeTy);
 
       // If there's a non-1 type size multiplier, then we can do the
       // signedness check at the same time as we do the multiply
       // because a negative number times anything will cause an
       // unsigned overflow.  Otherwise, we have to do it here. But at least
       // in this case, we can subsume the >= minElements check.
       if (typeSizeMultiplier == 1)
         hasOverflow = CGF.Builder.CreateICmpSLT(numElements,
                               llvm::ConstantInt::get(CGF.SizeTy, minElements));
 
     // Otherwise, zext up to size_t if necessary.
     } else if (numElementsWidth < sizeWidth) {
       numElements = CGF.Builder.CreateZExt(numElements, CGF.SizeTy);
     }
 
     assert(numElements->getType() == CGF.SizeTy);
 
     if (minElements) {
       // Don't allow allocation of fewer elements than we have initializers.
       if (!hasOverflow) {
         hasOverflow = CGF.Builder.CreateICmpULT(numElements,
                               llvm::ConstantInt::get(CGF.SizeTy, minElements));
       } else if (numElementsWidth > sizeWidth) {
         // The other existing overflow subsumes this check.
         // We do an unsigned comparison, since any signed value < -1 is
         // taken care of either above or below.
         hasOverflow = CGF.Builder.CreateOr(hasOverflow,
                           CGF.Builder.CreateICmpULT(numElements,
                               llvm::ConstantInt::get(CGF.SizeTy, minElements)));
       }
     }
 
     size = numElements;
 
     // Multiply by the type size if necessary.  This multiplier
     // includes all the factors for nested arrays.
     //
     // This step also causes numElements to be scaled up by the
     // nested-array factor if necessary.  Overflow on this computation
     // can be ignored because the result shouldn't be used if
     // allocation fails.
     if (typeSizeMultiplier != 1) {
       llvm::Function *umul_with_overflow
         = CGF.CGM.getIntrinsic(llvm::Intrinsic::umul_with_overflow, CGF.SizeTy);
 
       llvm::Value *tsmV =
         llvm::ConstantInt::get(CGF.SizeTy, typeSizeMultiplier);
       llvm::Value *result =
           CGF.Builder.CreateCall(umul_with_overflow, {size, tsmV});
 
       llvm::Value *overflowed = CGF.Builder.CreateExtractValue(result, 1);
       if (hasOverflow)
         hasOverflow = CGF.Builder.CreateOr(hasOverflow, overflowed);
       else
         hasOverflow = overflowed;
 
       size = CGF.Builder.CreateExtractValue(result, 0);
 
       // Also scale up numElements by the array size multiplier.
       if (arraySizeMultiplier != 1) {
         // If the base element type size is 1, then we can re-use the
         // multiply we just did.
         if (typeSize.isOne()) {
           assert(arraySizeMultiplier == typeSizeMultiplier);
           numElements = size;
 
         // Otherwise we need a separate multiply.
         } else {
           llvm::Value *asmV =
             llvm::ConstantInt::get(CGF.SizeTy, arraySizeMultiplier);
           numElements = CGF.Builder.CreateMul(numElements, asmV);
         }
       }
     } else {
       // numElements doesn't need to be scaled.
       assert(arraySizeMultiplier == 1);
     }
 
     // Add in the cookie size if necessary.
     if (cookieSize != 0) {
       sizeWithoutCookie = size;
 
       llvm::Function *uadd_with_overflow
         = CGF.CGM.getIntrinsic(llvm::Intrinsic::uadd_with_overflow, CGF.SizeTy);
 
       llvm::Value *cookieSizeV = llvm::ConstantInt::get(CGF.SizeTy, cookieSize);
       llvm::Value *result =
           CGF.Builder.CreateCall(uadd_with_overflow, {size, cookieSizeV});
 
       llvm::Value *overflowed = CGF.Builder.CreateExtractValue(result, 1);
       if (hasOverflow)
         hasOverflow = CGF.Builder.CreateOr(hasOverflow, overflowed);
       else
         hasOverflow = overflowed;
 
       size = CGF.Builder.CreateExtractValue(result, 0);
     }
 
     // If we had any possibility of dynamic overflow, make a select to
     // overwrite 'size' with an all-ones value, which should cause
     // operator new to throw.
     if (hasOverflow)
       size = CGF.Builder.CreateSelect(hasOverflow,
                                  llvm::Constant::getAllOnesValue(CGF.SizeTy),
                                       size);
   }
 
   if (cookieSize == 0)
     sizeWithoutCookie = size;
   else
     assert(sizeWithoutCookie && "didn't set sizeWithoutCookie?");
 
   return size;
 }
 
 static void StoreAnyExprIntoOneUnit(CodeGenFunction &CGF, const Expr *Init,
                                     QualType AllocType, Address NewPtr,
                                     AggValueSlot::Overlap_t MayOverlap) {
   // FIXME: Refactor with EmitExprAsInit.
   switch (CGF.getEvaluationKind(AllocType)) {
   case TEK_Scalar:
     CGF.EmitScalarInit(Init, nullptr,
                        CGF.MakeAddrLValue(NewPtr, AllocType), false);
     return;
   case TEK_Complex:
     CGF.EmitComplexExprIntoLValue(Init, CGF.MakeAddrLValue(NewPtr, AllocType),
                                   /*isInit*/ true);
     return;
   case TEK_Aggregate: {
     AggValueSlot Slot
       = AggValueSlot::forAddr(NewPtr, AllocType.getQualifiers(),
                               AggValueSlot::IsDestructed,
                               AggValueSlot::DoesNotNeedGCBarriers,
                               AggValueSlot::IsNotAliased,
                               MayOverlap, AggValueSlot::IsNotZeroed,
                               AggValueSlot::IsSanitizerChecked);
     CGF.EmitAggExpr(Init, Slot);
     return;
   }
   }
   llvm_unreachable("bad evaluation kind");
 }
 
 void CodeGenFunction::EmitNewArrayInitializer(
     const CXXNewExpr *E, QualType ElementType, llvm::Type *ElementTy,
     Address BeginPtr, llvm::Value *NumElements,
     llvm::Value *AllocSizeWithoutCookie) {
   // If we have a type with trivial initialization and no initializer,
   // there's nothing to do.
   if (!E->hasInitializer())
     return;
 
   Address CurPtr = BeginPtr;
 
   unsigned InitListElements = 0;
 
   const Expr *Init = E->getInitializer();
   Address EndOfInit = Address::invalid();
   QualType::DestructionKind DtorKind = ElementType.isDestructedType();
   EHScopeStack::stable_iterator Cleanup;
   llvm::Instruction *CleanupDominator = nullptr;
 
   CharUnits ElementSize = getContext().getTypeSizeInChars(ElementType);
   CharUnits ElementAlign =
     BeginPtr.getAlignment().alignmentOfArrayElement(ElementSize);
 
   // Attempt to perform zero-initialization using memset.
   auto TryMemsetInitialization = [&]() -> bool {
     // FIXME: If the type is a pointer-to-data-member under the Itanium ABI,
     // we can initialize with a memset to -1.
     if (!CGM.getTypes().isZeroInitializable(ElementType))
       return false;
 
     // Optimization: since zero initialization will just set the memory
     // to all zeroes, generate a single memset to do it in one shot.
 
     // Subtract out the size of any elements we've already initialized.
     auto *RemainingSize = AllocSizeWithoutCookie;
     if (InitListElements) {
       // We know this can't overflow; we check this when doing the allocation.
       auto *InitializedSize = llvm::ConstantInt::get(
           RemainingSize->getType(),
           getContext().getTypeSizeInChars(ElementType).getQuantity() *
               InitListElements);
       RemainingSize = Builder.CreateSub(RemainingSize, InitializedSize);
     }
 
     // Create the memset.
     Builder.CreateMemSet(CurPtr, Builder.getInt8(0), RemainingSize, false);
     return true;
   };
 
   // If the initializer is an initializer list, first do the explicit elements.
   if (const InitListExpr *ILE = dyn_cast<InitListExpr>(Init)) {
     // Initializing from a (braced) string literal is a special case; the init
     // list element does not initialize a (single) array element.
     if (ILE->isStringLiteralInit()) {
       // Initialize the initial portion of length equal to that of the string
       // literal. The allocation must be for at least this much; we emitted a
       // check for that earlier.
       AggValueSlot Slot =
           AggValueSlot::forAddr(CurPtr, ElementType.getQualifiers(),
                                 AggValueSlot::IsDestructed,
                                 AggValueSlot::DoesNotNeedGCBarriers,
                                 AggValueSlot::IsNotAliased,
                                 AggValueSlot::DoesNotOverlap,
                                 AggValueSlot::IsNotZeroed,
                                 AggValueSlot::IsSanitizerChecked);
       EmitAggExpr(ILE->getInit(0), Slot);
 
       // Move past these elements.
       InitListElements =
           cast<ConstantArrayType>(ILE->getType()->getAsArrayTypeUnsafe())
               ->getSize().getZExtValue();
       CurPtr = Builder.CreateConstInBoundsGEP(
           CurPtr, InitListElements, "string.init.end");
 
       // Zero out the rest, if any remain.
       llvm::ConstantInt *ConstNum = dyn_cast<llvm::ConstantInt>(NumElements);
       if (!ConstNum || !ConstNum->equalsInt(InitListElements)) {
         bool OK = TryMemsetInitialization();
         (void)OK;
         assert(OK && "couldn't memset character type?");
       }
       return;
     }
 
     InitListElements = ILE->getNumInits();
 
     // If this is a multi-dimensional array new, we will initialize multiple
     // elements with each init list element.
     QualType AllocType = E->getAllocatedType();
     if (const ConstantArrayType *CAT = dyn_cast_or_null<ConstantArrayType>(
             AllocType->getAsArrayTypeUnsafe())) {
       ElementTy = ConvertTypeForMem(AllocType);
       CurPtr = Builder.CreateElementBitCast(CurPtr, ElementTy);
       InitListElements *= getContext().getConstantArrayElementCount(CAT);
     }
 
     // Enter a partial-destruction Cleanup if necessary.
     if (needsEHCleanup(DtorKind)) {
       // In principle we could tell the Cleanup where we are more
       // directly, but the control flow can get so varied here that it
       // would actually be quite complex.  Therefore we go through an
       // alloca.
       EndOfInit = CreateTempAlloca(BeginPtr.getType(), getPointerAlign(),
                                    "array.init.end");
       CleanupDominator = Builder.CreateStore(BeginPtr.getPointer(), EndOfInit);
       pushIrregularPartialArrayCleanup(BeginPtr.getPointer(), EndOfInit,
                                        ElementType, ElementAlign,
                                        getDestroyer(DtorKind));
       Cleanup = EHStack.stable_begin();
     }
 
     CharUnits StartAlign = CurPtr.getAlignment();
     for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i) {
       // Tell the cleanup that it needs to destroy up to this
       // element.  TODO: some of these stores can be trivially
       // observed to be unnecessary.
       if (EndOfInit.isValid()) {
         auto FinishedPtr =
           Builder.CreateBitCast(CurPtr.getPointer(), BeginPtr.getType());
         Builder.CreateStore(FinishedPtr, EndOfInit);
       }
       // FIXME: If the last initializer is an incomplete initializer list for
       // an array, and we have an array filler, we can fold together the two
       // initialization loops.
       StoreAnyExprIntoOneUnit(*this, ILE->getInit(i),
                               ILE->getInit(i)->getType(), CurPtr,
                               AggValueSlot::DoesNotOverlap);
       CurPtr = Address(Builder.CreateInBoundsGEP(
                            CurPtr.getElementType(), CurPtr.getPointer(),
                            Builder.getSize(1), "array.exp.next"),
                        CurPtr.getElementType(),
                        StartAlign.alignmentAtOffset((i + 1) * ElementSize));
     }
 
     // The remaining elements are filled with the array filler expression.
     Init = ILE->getArrayFiller();
 
     // Extract the initializer for the individual array elements by pulling
     // out the array filler from all the nested initializer lists. This avoids
     // generating a nested loop for the initialization.
     while (Init && Init->getType()->isConstantArrayType()) {
       auto *SubILE = dyn_cast<InitListExpr>(Init);
       if (!SubILE)
         break;
       assert(SubILE->getNumInits() == 0 && "explicit inits in array filler?");
       Init = SubILE->getArrayFiller();
     }
 
     // Switch back to initializing one base element at a time.
     CurPtr = Builder.CreateElementBitCast(CurPtr, BeginPtr.getElementType());
   }
 
   // If all elements have already been initialized, skip any further
   // initialization.
   llvm::ConstantInt *ConstNum = dyn_cast<llvm::ConstantInt>(NumElements);
   if (ConstNum && ConstNum->getZExtValue() <= InitListElements) {
     // If there was a Cleanup, deactivate it.
     if (CleanupDominator)
       DeactivateCleanupBlock(Cleanup, CleanupDominator);
     return;
   }
 
   assert(Init && "have trailing elements to initialize but no initializer");
 
   // If this is a constructor call, try to optimize it out, and failing that
   // emit a single loop to initialize all remaining elements.
   if (const CXXConstructExpr *CCE = dyn_cast<CXXConstructExpr>(Init)) {
     CXXConstructorDecl *Ctor = CCE->getConstructor();
     if (Ctor->isTrivial()) {
       // If new expression did not specify value-initialization, then there
       // is no initialization.
       if (!CCE->requiresZeroInitialization() || Ctor->getParent()->isEmpty())
         return;
 
       if (TryMemsetInitialization())
         return;
     }
 
     // Store the new Cleanup position for irregular Cleanups.
     //
     // FIXME: Share this cleanup with the constructor call emission rather than
     // having it create a cleanup of its own.
     if (EndOfInit.isValid())
       Builder.CreateStore(CurPtr.getPointer(), EndOfInit);
 
     // Emit a constructor call loop to initialize the remaining elements.
     if (InitListElements)
       NumElements = Builder.CreateSub(
           NumElements,
           llvm::ConstantInt::get(NumElements->getType(), InitListElements));
     EmitCXXAggrConstructorCall(Ctor, NumElements, CurPtr, CCE,
                                /*NewPointerIsChecked*/true,
                                CCE->requiresZeroInitialization());
     return;
   }
 
   // If this is value-initialization, we can usually use memset.
   ImplicitValueInitExpr IVIE(ElementType);
   if (isa<ImplicitValueInitExpr>(Init)) {
     if (TryMemsetInitialization())
       return;
 
     // Switch to an ImplicitValueInitExpr for the element type. This handles
     // only one case: multidimensional array new of pointers to members. In
     // all other cases, we already have an initializer for the array element.
     Init = &IVIE;
   }
 
   // At this point we should have found an initializer for the individual
   // elements of the array.
   assert(getContext().hasSameUnqualifiedType(ElementType, Init->getType()) &&
          "got wrong type of element to initialize");
 
   // If we have an empty initializer list, we can usually use memset.
   if (auto *ILE = dyn_cast<InitListExpr>(Init))
     if (ILE->getNumInits() == 0 && TryMemsetInitialization())
       return;
 
   // If we have a struct whose every field is value-initialized, we can
   // usually use memset.
   if (auto *ILE = dyn_cast<InitListExpr>(Init)) {
     if (const RecordType *RType = ILE->getType()->getAs<RecordType>()) {
       if (RType->getDecl()->isStruct()) {
         unsigned NumElements = 0;
         if (auto *CXXRD = dyn_cast<CXXRecordDecl>(RType->getDecl()))
           NumElements = CXXRD->getNumBases();
         for (auto *Field : RType->getDecl()->fields())
           if (!Field->isUnnamedBitfield())
             ++NumElements;
         // FIXME: Recurse into nested InitListExprs.
         if (ILE->getNumInits() == NumElements)
           for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i)
             if (!isa<ImplicitValueInitExpr>(ILE->getInit(i)))
               --NumElements;
         if (ILE->getNumInits() == NumElements && TryMemsetInitialization())
           return;
       }
     }
   }
 
   // Create the loop blocks.
   llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
   llvm::BasicBlock *LoopBB = createBasicBlock("new.loop");
   llvm::BasicBlock *ContBB = createBasicBlock("new.loop.end");
 
   // Find the end of the array, hoisted out of the loop.
   llvm::Value *EndPtr =
     Builder.CreateInBoundsGEP(BeginPtr.getElementType(), BeginPtr.getPointer(),
                               NumElements, "array.end");
 
   // If the number of elements isn't constant, we have to now check if there is
   // anything left to initialize.
   if (!ConstNum) {
     llvm::Value *IsEmpty =
       Builder.CreateICmpEQ(CurPtr.getPointer(), EndPtr, "array.isempty");
     Builder.CreateCondBr(IsEmpty, ContBB, LoopBB);
   }
 
   // Enter the loop.
   EmitBlock(LoopBB);
 
   // Set up the current-element phi.
   llvm::PHINode *CurPtrPhi =
     Builder.CreatePHI(CurPtr.getType(), 2, "array.cur");
   CurPtrPhi->addIncoming(CurPtr.getPointer(), EntryBB);
 
   CurPtr = Address(CurPtrPhi, ElementAlign);
 
   // Store the new Cleanup position for irregular Cleanups.
   if (EndOfInit.isValid())
     Builder.CreateStore(CurPtr.getPointer(), EndOfInit);
 
   // Enter a partial-destruction Cleanup if necessary.
   if (!CleanupDominator && needsEHCleanup(DtorKind)) {
     pushRegularPartialArrayCleanup(BeginPtr.getPointer(), CurPtr.getPointer(),
                                    ElementType, ElementAlign,
                                    getDestroyer(DtorKind));
     Cleanup = EHStack.stable_begin();
     CleanupDominator = Builder.CreateUnreachable();
   }
 
   // Emit the initializer into this element.
   StoreAnyExprIntoOneUnit(*this, Init, Init->getType(), CurPtr,
                           AggValueSlot::DoesNotOverlap);
 
   // Leave the Cleanup if we entered one.
   if (CleanupDominator) {
     DeactivateCleanupBlock(Cleanup, CleanupDominator);
     CleanupDominator->eraseFromParent();
   }
 
   // Advance to the next element by adjusting the pointer type as necessary.
   llvm::Value *NextPtr =
     Builder.CreateConstInBoundsGEP1_32(ElementTy, CurPtr.getPointer(), 1,
                                        "array.next");
 
   // Check whether we've gotten to the end of the array and, if so,
   // exit the loop.
   llvm::Value *IsEnd = Builder.CreateICmpEQ(NextPtr, EndPtr, "array.atend");
   Builder.CreateCondBr(IsEnd, ContBB, LoopBB);
   CurPtrPhi->addIncoming(NextPtr, Builder.GetInsertBlock());
 
   EmitBlock(ContBB);
 }
 
 static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E,
                                QualType ElementType, llvm::Type *ElementTy,
                                Address NewPtr, llvm::Value *NumElements,
                                llvm::Value *AllocSizeWithoutCookie) {
   ApplyDebugLocation DL(CGF, E);
   if (E->isArray())
     CGF.EmitNewArrayInitializer(E, ElementType, ElementTy, NewPtr, NumElements,
                                 AllocSizeWithoutCookie);
   else if (const Expr *Init = E->getInitializer())
     StoreAnyExprIntoOneUnit(CGF, Init, E->getAllocatedType(), NewPtr,
                             AggValueSlot::DoesNotOverlap);
 }
 
 /// Emit a call to an operator new or operator delete function, as implicitly
 /// created by new-expressions and delete-expressions.
 static RValue EmitNewDeleteCall(CodeGenFunction &CGF,
                                 const FunctionDecl *CalleeDecl,
                                 const FunctionProtoType *CalleeType,
                                 const CallArgList &Args) {
   llvm::CallBase *CallOrInvoke;
   llvm::Constant *CalleePtr = CGF.CGM.GetAddrOfFunction(CalleeDecl);
   CGCallee Callee = CGCallee::forDirect(CalleePtr, GlobalDecl(CalleeDecl));
   RValue RV =
       CGF.EmitCall(CGF.CGM.getTypes().arrangeFreeFunctionCall(
                        Args, CalleeType, /*ChainCall=*/false),
                    Callee, ReturnValueSlot(), Args, &CallOrInvoke);
 
   /// C++1y [expr.new]p10:
   ///   [In a new-expression,] an implementation is allowed to omit a call
   ///   to a replaceable global allocation function.
   ///
   /// We model such elidable calls with the 'builtin' attribute.
   llvm::Function *Fn = dyn_cast<llvm::Function>(CalleePtr);
   if (CalleeDecl->isReplaceableGlobalAllocationFunction() &&
       Fn && Fn->hasFnAttribute(llvm::Attribute::NoBuiltin)) {
     CallOrInvoke->addFnAttr(llvm::Attribute::Builtin);
   }
 
   return RV;
 }
 
 RValue CodeGenFunction::EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
                                                  const CallExpr *TheCall,
                                                  bool IsDelete) {
   CallArgList Args;
   EmitCallArgs(Args, Type, TheCall->arguments());
   // Find the allocation or deallocation function that we're calling.
   ASTContext &Ctx = getContext();
   DeclarationName Name = Ctx.DeclarationNames
       .getCXXOperatorName(IsDelete ? OO_Delete : OO_New);
 
   for (auto *Decl : Ctx.getTranslationUnitDecl()->lookup(Name))
     if (auto *FD = dyn_cast<FunctionDecl>(Decl))
       if (Ctx.hasSameType(FD->getType(), QualType(Type, 0)))
         return EmitNewDeleteCall(*this, FD, Type, Args);
   llvm_unreachable("predeclared global operator new/delete is missing");
 }
 
 namespace {
 /// The parameters to pass to a usual operator delete.
 struct UsualDeleteParams {
   bool DestroyingDelete = false;
   bool Size = false;
   bool Alignment = false;
 };
 }
 
 static UsualDeleteParams getUsualDeleteParams(const FunctionDecl *FD) {
   UsualDeleteParams Params;
 
   const FunctionProtoType *FPT = FD->getType()->castAs<FunctionProtoType>();
   auto AI = FPT->param_type_begin(), AE = FPT->param_type_end();
 
   // The first argument is always a void*.
   ++AI;
 
   // The next parameter may be a std::destroying_delete_t.
   if (FD->isDestroyingOperatorDelete()) {
     Params.DestroyingDelete = true;
     assert(AI != AE);
     ++AI;
   }
 
   // Figure out what other parameters we should be implicitly passing.
   if (AI != AE && (*AI)->isIntegerType()) {
     Params.Size = true;
     ++AI;
   }
 
   if (AI != AE && (*AI)->isAlignValT()) {
     Params.Alignment = true;
     ++AI;
   }
 
   assert(AI == AE && "unexpected usual deallocation function parameter");
   return Params;
 }
 
 namespace {
   /// A cleanup to call the given 'operator delete' function upon abnormal
   /// exit from a new expression. Templated on a traits type that deals with
   /// ensuring that the arguments dominate the cleanup if necessary.
   template<typename Traits>
   class CallDeleteDuringNew final : public EHScopeStack::Cleanup {
     /// Type used to hold llvm::Value*s.
     typedef typename Traits::ValueTy ValueTy;
     /// Type used to hold RValues.
     typedef typename Traits::RValueTy RValueTy;
     struct PlacementArg {
       RValueTy ArgValue;
       QualType ArgType;
     };
 
     unsigned NumPlacementArgs : 31;
     unsigned PassAlignmentToPlacementDelete : 1;
     const FunctionDecl *OperatorDelete;
     ValueTy Ptr;
     ValueTy AllocSize;
     CharUnits AllocAlign;
 
     PlacementArg *getPlacementArgs() {
       return reinterpret_cast<PlacementArg *>(this + 1);
     }
 
   public:
     static size_t getExtraSize(size_t NumPlacementArgs) {
       return NumPlacementArgs * sizeof(PlacementArg);
     }
 
     CallDeleteDuringNew(size_t NumPlacementArgs,
                         const FunctionDecl *OperatorDelete, ValueTy Ptr,
                         ValueTy AllocSize, bool PassAlignmentToPlacementDelete,
                         CharUnits AllocAlign)
       : NumPlacementArgs(NumPlacementArgs),
         PassAlignmentToPlacementDelete(PassAlignmentToPlacementDelete),
         OperatorDelete(OperatorDelete), Ptr(Ptr), AllocSize(AllocSize),
         AllocAlign(AllocAlign) {}
 
     void setPlacementArg(unsigned I, RValueTy Arg, QualType Type) {
       assert(I < NumPlacementArgs && "index out of range");
       getPlacementArgs()[I] = {Arg, Type};
     }
 
     void Emit(CodeGenFunction &CGF, Flags flags) override {
       const auto *FPT = OperatorDelete->getType()->castAs<FunctionProtoType>();
       CallArgList DeleteArgs;
 
       // The first argument is always a void* (or C* for a destroying operator
       // delete for class type C).
       DeleteArgs.add(Traits::get(CGF, Ptr), FPT->getParamType(0));
 
       // Figure out what other parameters we should be implicitly passing.
       UsualDeleteParams Params;
       if (NumPlacementArgs) {
         // A placement deallocation function is implicitly passed an alignment
         // if the placement allocation function was, but is never passed a size.
         Params.Alignment = PassAlignmentToPlacementDelete;
       } else {
         // For a non-placement new-expression, 'operator delete' can take a
         // size and/or an alignment if it has the right parameters.
         Params = getUsualDeleteParams(OperatorDelete);
       }
 
       assert(!Params.DestroyingDelete &&
              "should not call destroying delete in a new-expression");
 
       // The second argument can be a std::size_t (for non-placement delete).
       if (Params.Size)
         DeleteArgs.add(Traits::get(CGF, AllocSize),
                        CGF.getContext().getSizeType());
 
       // The next (second or third) argument can be a std::align_val_t, which
       // is an enum whose underlying type is std::size_t.
       // FIXME: Use the right type as the parameter type. Note that in a call
       // to operator delete(size_t, ...), we may not have it available.
       if (Params.Alignment)
         DeleteArgs.add(RValue::get(llvm::ConstantInt::get(
                            CGF.SizeTy, AllocAlign.getQuantity())),
                        CGF.getContext().getSizeType());
 
       // Pass the rest of the arguments, which must match exactly.
       for (unsigned I = 0; I != NumPlacementArgs; ++I) {
         auto Arg = getPlacementArgs()[I];
         DeleteArgs.add(Traits::get(CGF, Arg.ArgValue), Arg.ArgType);
       }
 
       // Call 'operator delete'.
       EmitNewDeleteCall(CGF, OperatorDelete, FPT, DeleteArgs);
     }
   };
 }
 
 /// Enter a cleanup to call 'operator delete' if the initializer in a
 /// new-expression throws.
 static void EnterNewDeleteCleanup(CodeGenFunction &CGF,
                                   const CXXNewExpr *E,
                                   Address NewPtr,
                                   llvm::Value *AllocSize,
                                   CharUnits AllocAlign,
                                   const CallArgList &NewArgs) {
   unsigned NumNonPlacementArgs = E->passAlignment() ? 2 : 1;
 
   // If we're not inside a conditional branch, then the cleanup will
   // dominate and we can do the easier (and more efficient) thing.
   if (!CGF.isInConditionalBranch()) {
     struct DirectCleanupTraits {
       typedef llvm::Value *ValueTy;
       typedef RValue RValueTy;
       static RValue get(CodeGenFunction &, ValueTy V) { return RValue::get(V); }
       static RValue get(CodeGenFunction &, RValueTy V) { return V; }
     };
 
     typedef CallDeleteDuringNew<DirectCleanupTraits> DirectCleanup;
 
     DirectCleanup *Cleanup = CGF.EHStack
       .pushCleanupWithExtra<DirectCleanup>(EHCleanup,
                                            E->getNumPlacementArgs(),
                                            E->getOperatorDelete(),
                                            NewPtr.getPointer(),
                                            AllocSize,
                                            E->passAlignment(),
                                            AllocAlign);
     for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) {
       auto &Arg = NewArgs[I + NumNonPlacementArgs];
       Cleanup->setPlacementArg(I, Arg.getRValue(CGF), Arg.Ty);
     }
 
     return;
   }
 
   // Otherwise, we need to save all this stuff.
   DominatingValue<RValue>::saved_type SavedNewPtr =
     DominatingValue<RValue>::save(CGF, RValue::get(NewPtr.getPointer()));
   DominatingValue<RValue>::saved_type SavedAllocSize =
     DominatingValue<RValue>::save(CGF, RValue::get(AllocSize));
 
   struct ConditionalCleanupTraits {
     typedef DominatingValue<RValue>::saved_type ValueTy;
     typedef DominatingValue<RValue>::saved_type RValueTy;
     static RValue get(CodeGenFunction &CGF, ValueTy V) {
       return V.restore(CGF);
     }
   };
   typedef CallDeleteDuringNew<ConditionalCleanupTraits> ConditionalCleanup;
 
   ConditionalCleanup *Cleanup = CGF.EHStack
     .pushCleanupWithExtra<ConditionalCleanup>(EHCleanup,
                                               E->getNumPlacementArgs(),
                                               E->getOperatorDelete(),
                                               SavedNewPtr,
                                               SavedAllocSize,
                                               E->passAlignment(),
                                               AllocAlign);
   for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) {
     auto &Arg = NewArgs[I + NumNonPlacementArgs];
     Cleanup->setPlacementArg(
         I, DominatingValue<RValue>::save(CGF, Arg.getRValue(CGF)), Arg.Ty);
   }
 
   CGF.initFullExprCleanup();
 }
 
 llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
   // The element type being allocated.
   QualType allocType = getContext().getBaseElementType(E->getAllocatedType());
 
   // 1. Build a call to the allocation function.
   FunctionDecl *allocator = E->getOperatorNew();
 
   // If there is a brace-initializer, cannot allocate fewer elements than inits.
   unsigned minElements = 0;
   if (E->isArray() && E->hasInitializer()) {
     const InitListExpr *ILE = dyn_cast<InitListExpr>(E->getInitializer());
     if (ILE && ILE->isStringLiteralInit())
       minElements =
           cast<ConstantArrayType>(ILE->getType()->getAsArrayTypeUnsafe())
               ->getSize().getZExtValue();
     else if (ILE)
       minElements = ILE->getNumInits();
   }
 
   llvm::Value *numElements = nullptr;
   llvm::Value *allocSizeWithoutCookie = nullptr;
   llvm::Value *allocSize =
     EmitCXXNewAllocSize(*this, E, minElements, numElements,
                         allocSizeWithoutCookie);
-  CharUnits allocAlign = getContext().getPreferredTypeAlignInChars(allocType);
+  CharUnits allocAlign = getContext().getTypeAlignInChars(allocType);
 
   // Emit the allocation call.  If the allocator is a global placement
   // operator, just "inline" it directly.
   Address allocation = Address::invalid();
   CallArgList allocatorArgs;
   if (allocator->isReservedGlobalPlacementOperator()) {
     assert(E->getNumPlacementArgs() == 1);
     const Expr *arg = *E->placement_arguments().begin();
 
     LValueBaseInfo BaseInfo;
     allocation = EmitPointerWithAlignment(arg, &BaseInfo);
 
     // The pointer expression will, in many cases, be an opaque void*.
     // In these cases, discard the computed alignment and use the
     // formal alignment of the allocated type.
     if (BaseInfo.getAlignmentSource() != AlignmentSource::Decl)
       allocation = allocation.withAlignment(allocAlign);
 
     // Set up allocatorArgs for the call to operator delete if it's not
     // the reserved global operator.
     if (E->getOperatorDelete() &&
         !E->getOperatorDelete()->isReservedGlobalPlacementOperator()) {
       allocatorArgs.add(RValue::get(allocSize), getContext().getSizeType());
       allocatorArgs.add(RValue::get(allocation.getPointer()), arg->getType());
     }
 
   } else {
     const FunctionProtoType *allocatorType =
       allocator->getType()->castAs<FunctionProtoType>();
     unsigned ParamsToSkip = 0;
 
     // The allocation size is the first argument.
     QualType sizeType = getContext().getSizeType();
     allocatorArgs.add(RValue::get(allocSize), sizeType);
     ++ParamsToSkip;
 
     if (allocSize != allocSizeWithoutCookie) {
       CharUnits cookieAlign = getSizeAlign(); // FIXME: Ask the ABI.
       allocAlign = std::max(allocAlign, cookieAlign);
     }
 
     // The allocation alignment may be passed as the second argument.
     if (E->passAlignment()) {
       QualType AlignValT = sizeType;
       if (allocatorType->getNumParams() > 1) {
         AlignValT = allocatorType->getParamType(1);
         assert(getContext().hasSameUnqualifiedType(
                    AlignValT->castAs<EnumType>()->getDecl()->getIntegerType(),
                    sizeType) &&
                "wrong type for alignment parameter");
         ++ParamsToSkip;
       } else {
         // Corner case, passing alignment to 'operator new(size_t, ...)'.
         assert(allocator->isVariadic() && "can't pass alignment to allocator");
       }
       allocatorArgs.add(
           RValue::get(llvm::ConstantInt::get(SizeTy, allocAlign.getQuantity())),
           AlignValT);
     }
 
     // FIXME: Why do we not pass a CalleeDecl here?
     EmitCallArgs(allocatorArgs, allocatorType, E->placement_arguments(),
                  /*AC*/AbstractCallee(), /*ParamsToSkip*/ParamsToSkip);
 
     RValue RV =
       EmitNewDeleteCall(*this, allocator, allocatorType, allocatorArgs);
 
     // Set !heapallocsite metadata on the call to operator new.
     if (getDebugInfo())
       if (auto *newCall = dyn_cast<llvm::CallBase>(RV.getScalarVal()))
         getDebugInfo()->addHeapAllocSiteMetadata(newCall, allocType,
                                                  E->getExprLoc());
 
     // If this was a call to a global replaceable allocation function that does
     // not take an alignment argument, the allocator is known to produce
     // storage that's suitably aligned for any object that fits, up to a known
     // threshold. Otherwise assume it's suitably aligned for the allocated type.
     CharUnits allocationAlign = allocAlign;
     if (!E->passAlignment() &&
         allocator->isReplaceableGlobalAllocationFunction()) {
       unsigned AllocatorAlign = llvm::PowerOf2Floor(std::min<uint64_t>(
           Target.getNewAlign(), getContext().getTypeSize(allocType)));
       allocationAlign = std::max(
           allocationAlign, getContext().toCharUnitsFromBits(AllocatorAlign));
     }
 
     allocation = Address(RV.getScalarVal(), Int8Ty, allocationAlign);
   }
 
   // Emit a null check on the allocation result if the allocation
   // function is allowed to return null (because it has a non-throwing
   // exception spec or is the reserved placement new) and we have an
   // interesting initializer will be running sanitizers on the initialization.
   bool nullCheck = E->shouldNullCheckAllocation() &&
                    (!allocType.isPODType(getContext()) || E->hasInitializer() ||
                     sanitizePerformTypeCheck());
 
   llvm::BasicBlock *nullCheckBB = nullptr;
   llvm::BasicBlock *contBB = nullptr;
 
   // The null-check means that the initializer is conditionally
   // evaluated.
   ConditionalEvaluation conditional(*this);
 
   if (nullCheck) {
     conditional.begin(*this);
 
     nullCheckBB = Builder.GetInsertBlock();
     llvm::BasicBlock *notNullBB = createBasicBlock("new.notnull");
     contBB = createBasicBlock("new.cont");
 
     llvm::Value *isNull =
       Builder.CreateIsNull(allocation.getPointer(), "new.isnull");
     Builder.CreateCondBr(isNull, contBB, notNullBB);
     EmitBlock(notNullBB);
   }
 
   // If there's an operator delete, enter a cleanup to call it if an
   // exception is thrown.
   EHScopeStack::stable_iterator operatorDeleteCleanup;
   llvm::Instruction *cleanupDominator = nullptr;
   if (E->getOperatorDelete() &&
       !E->getOperatorDelete()->isReservedGlobalPlacementOperator()) {
     EnterNewDeleteCleanup(*this, E, allocation, allocSize, allocAlign,
                           allocatorArgs);
     operatorDeleteCleanup = EHStack.stable_begin();
     cleanupDominator = Builder.CreateUnreachable();
   }
 
   assert((allocSize == allocSizeWithoutCookie) ==
          CalculateCookiePadding(*this, E).isZero());
   if (allocSize != allocSizeWithoutCookie) {
     assert(E->isArray());
     allocation = CGM.getCXXABI().InitializeArrayCookie(*this, allocation,
                                                        numElements,
                                                        E, allocType);
   }
 
   llvm::Type *elementTy = ConvertTypeForMem(allocType);
   Address result = Builder.CreateElementBitCast(allocation, elementTy);
 
   // Passing pointer through launder.invariant.group to avoid propagation of
   // vptrs information which may be included in previous type.
   // To not break LTO with different optimizations levels, we do it regardless
   // of optimization level.
   if (CGM.getCodeGenOpts().StrictVTablePointers &&
       allocator->isReservedGlobalPlacementOperator())
     result = Builder.CreateLaunderInvariantGroup(result);
 
   // Emit sanitizer checks for pointer value now, so that in the case of an
   // array it was checked only once and not at each constructor call. We may
   // have already checked that the pointer is non-null.
   // FIXME: If we have an array cookie and a potentially-throwing allocator,
   // we'll null check the wrong pointer here.
   SanitizerSet SkippedChecks;
   SkippedChecks.set(SanitizerKind::Null, nullCheck);
   EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall,
                 E->getAllocatedTypeSourceInfo()->getTypeLoc().getBeginLoc(),
                 result.getPointer(), allocType, result.getAlignment(),
                 SkippedChecks, numElements);
 
   EmitNewInitializer(*this, E, allocType, elementTy, result, numElements,
                      allocSizeWithoutCookie);
   if (E->isArray()) {
     // NewPtr is a pointer to the base element type.  If we're
     // allocating an array of arrays, we'll need to cast back to the
     // array pointer type.
     llvm::Type *resultType = ConvertTypeForMem(E->getType());
     if (result.getType() != resultType)
       result = Builder.CreateBitCast(result, resultType);
   }
 
   // Deactivate the 'operator delete' cleanup if we finished
   // initialization.
   if (operatorDeleteCleanup.isValid()) {
     DeactivateCleanupBlock(operatorDeleteCleanup, cleanupDominator);
     cleanupDominator->eraseFromParent();
   }
 
   llvm::Value *resultPtr = result.getPointer();
   if (nullCheck) {
     conditional.end(*this);
 
     llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
     EmitBlock(contBB);
 
     llvm::PHINode *PHI = Builder.CreatePHI(resultPtr->getType(), 2);
     PHI->addIncoming(resultPtr, notNullBB);
     PHI->addIncoming(llvm::Constant::getNullValue(resultPtr->getType()),
                      nullCheckBB);
 
     resultPtr = PHI;
   }
 
   return resultPtr;
 }
 
 void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD,
                                      llvm::Value *Ptr, QualType DeleteTy,
                                      llvm::Value *NumElements,
                                      CharUnits CookieSize) {
   assert((!NumElements && CookieSize.isZero()) ||
          DeleteFD->getOverloadedOperator() == OO_Array_Delete);
 
   const auto *DeleteFTy = DeleteFD->getType()->castAs<FunctionProtoType>();
   CallArgList DeleteArgs;
 
   auto Params = getUsualDeleteParams(DeleteFD);
   auto ParamTypeIt = DeleteFTy->param_type_begin();
 
   // Pass the pointer itself.
   QualType ArgTy = *ParamTypeIt++;
   llvm::Value *DeletePtr = Builder.CreateBitCast(Ptr, ConvertType(ArgTy));
   DeleteArgs.add(RValue::get(DeletePtr), ArgTy);
 
   // Pass the std::destroying_delete tag if present.
   llvm::AllocaInst *DestroyingDeleteTag = nullptr;
   if (Params.DestroyingDelete) {
     QualType DDTag = *ParamTypeIt++;
     llvm::Type *Ty = getTypes().ConvertType(DDTag);
     CharUnits Align = CGM.getNaturalTypeAlignment(DDTag);
     DestroyingDeleteTag = CreateTempAlloca(Ty, "destroying.delete.tag");
     DestroyingDeleteTag->setAlignment(Align.getAsAlign());
     DeleteArgs.add(RValue::getAggregate(Address(DestroyingDeleteTag, Align)), DDTag);
   }
 
   // Pass the size if the delete function has a size_t parameter.
   if (Params.Size) {
     QualType SizeType = *ParamTypeIt++;
     CharUnits DeleteTypeSize = getContext().getTypeSizeInChars(DeleteTy);
     llvm::Value *Size = llvm::ConstantInt::get(ConvertType(SizeType),
                                                DeleteTypeSize.getQuantity());
 
     // For array new, multiply by the number of elements.
     if (NumElements)
       Size = Builder.CreateMul(Size, NumElements);
 
     // If there is a cookie, add the cookie size.
     if (!CookieSize.isZero())
       Size = Builder.CreateAdd(
           Size, llvm::ConstantInt::get(SizeTy, CookieSize.getQuantity()));
 
     DeleteArgs.add(RValue::get(Size), SizeType);
   }
 
   // Pass the alignment if the delete function has an align_val_t parameter.
   if (Params.Alignment) {
     QualType AlignValType = *ParamTypeIt++;
     CharUnits DeleteTypeAlign =
         getContext().toCharUnitsFromBits(getContext().getTypeAlignIfKnown(
             DeleteTy, true /* NeedsPreferredAlignment */));
     llvm::Value *Align = llvm::ConstantInt::get(ConvertType(AlignValType),
                                                 DeleteTypeAlign.getQuantity());
     DeleteArgs.add(RValue::get(Align), AlignValType);
   }
 
   assert(ParamTypeIt == DeleteFTy->param_type_end() &&
          "unknown parameter to usual delete function");
 
   // Emit the call to delete.
   EmitNewDeleteCall(*this, DeleteFD, DeleteFTy, DeleteArgs);
 
   // If call argument lowering didn't use the destroying_delete_t alloca,
   // remove it again.
   if (DestroyingDeleteTag && DestroyingDeleteTag->use_empty())
     DestroyingDeleteTag->eraseFromParent();
 }
 
 namespace {
   /// Calls the given 'operator delete' on a single object.
   struct CallObjectDelete final : EHScopeStack::Cleanup {
     llvm::Value *Ptr;
     const FunctionDecl *OperatorDelete;
     QualType ElementType;
 
     CallObjectDelete(llvm::Value *Ptr,
                      const FunctionDecl *OperatorDelete,
                      QualType ElementType)
       : Ptr(Ptr), OperatorDelete(OperatorDelete), ElementType(ElementType) {}
 
     void Emit(CodeGenFunction &CGF, Flags flags) override {
       CGF.EmitDeleteCall(OperatorDelete, Ptr, ElementType);
     }
   };
 }
 
 void
 CodeGenFunction::pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete,
                                              llvm::Value *CompletePtr,
                                              QualType ElementType) {
   EHStack.pushCleanup<CallObjectDelete>(NormalAndEHCleanup, CompletePtr,
                                         OperatorDelete, ElementType);
 }
 
 /// Emit the code for deleting a single object with a destroying operator
 /// delete. If the element type has a non-virtual destructor, Ptr has already
 /// been converted to the type of the parameter of 'operator delete'. Otherwise
 /// Ptr points to an object of the static type.
 static void EmitDestroyingObjectDelete(CodeGenFunction &CGF,
                                        const CXXDeleteExpr *DE, Address Ptr,
                                        QualType ElementType) {
   auto *Dtor = ElementType->getAsCXXRecordDecl()->getDestructor();
   if (Dtor && Dtor->isVirtual())
     CGF.CGM.getCXXABI().emitVirtualObjectDelete(CGF, DE, Ptr, ElementType,
                                                 Dtor);
   else
     CGF.EmitDeleteCall(DE->getOperatorDelete(), Ptr.getPointer(), ElementType);
 }
 
 /// Emit the code for deleting a single object.
 /// \return \c true if we started emitting UnconditionalDeleteBlock, \c false
 /// if not.
 static bool EmitObjectDelete(CodeGenFunction &CGF,
                              const CXXDeleteExpr *DE,
                              Address Ptr,
                              QualType ElementType,
                              llvm::BasicBlock *UnconditionalDeleteBlock) {
   // C++11 [expr.delete]p3:
   //   If the static type of the object to be deleted is different from its
   //   dynamic type, the static type shall be a base class of the dynamic type
   //   of the object to be deleted and the static type shall have a virtual
   //   destructor or the behavior is undefined.
   CGF.EmitTypeCheck(CodeGenFunction::TCK_MemberCall,
                     DE->getExprLoc(), Ptr.getPointer(),
                     ElementType);
 
   const FunctionDecl *OperatorDelete = DE->getOperatorDelete();
   assert(!OperatorDelete->isDestroyingOperatorDelete());
 
   // Find the destructor for the type, if applicable.  If the
   // destructor is virtual, we'll just emit the vcall and return.
   const CXXDestructorDecl *Dtor = nullptr;
   if (const RecordType *RT = ElementType->getAs<RecordType>()) {
     CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
     if (RD->hasDefinition() && !RD->hasTrivialDestructor()) {
       Dtor = RD->getDestructor();
 
       if (Dtor->isVirtual()) {
         bool UseVirtualCall = true;
         const Expr *Base = DE->getArgument();
         if (auto *DevirtualizedDtor =
                 dyn_cast_or_null<const CXXDestructorDecl>(
                     Dtor->getDevirtualizedMethod(
                         Base, CGF.CGM.getLangOpts().AppleKext))) {
           UseVirtualCall = false;
           const CXXRecordDecl *DevirtualizedClass =
               DevirtualizedDtor->getParent();
           if (declaresSameEntity(getCXXRecord(Base), DevirtualizedClass)) {
             // Devirtualized to the class of the base type (the type of the
             // whole expression).
             Dtor = DevirtualizedDtor;
           } else {
             // Devirtualized to some other type. Would need to cast the this
             // pointer to that type but we don't have support for that yet, so
             // do a virtual call. FIXME: handle the case where it is
             // devirtualized to the derived type (the type of the inner
             // expression) as in EmitCXXMemberOrOperatorMemberCallExpr.
             UseVirtualCall = true;
           }
         }
         if (UseVirtualCall) {
           CGF.CGM.getCXXABI().emitVirtualObjectDelete(CGF, DE, Ptr, ElementType,
                                                       Dtor);
           return false;
         }
       }
     }
   }
 
   // Make sure that we call delete even if the dtor throws.
   // This doesn't have to a conditional cleanup because we're going
   // to pop it off in a second.
   CGF.EHStack.pushCleanup<CallObjectDelete>(NormalAndEHCleanup,
                                             Ptr.getPointer(),
                                             OperatorDelete, ElementType);
 
   if (Dtor)
     CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
                               /*ForVirtualBase=*/false,
                               /*Delegating=*/false,
                               Ptr, ElementType);
   else if (auto Lifetime = ElementType.getObjCLifetime()) {
     switch (Lifetime) {
     case Qualifiers::OCL_None:
     case Qualifiers::OCL_ExplicitNone:
     case Qualifiers::OCL_Autoreleasing:
       break;
 
     case Qualifiers::OCL_Strong:
       CGF.EmitARCDestroyStrong(Ptr, ARCPreciseLifetime);
       break;
 
     case Qualifiers::OCL_Weak:
       CGF.EmitARCDestroyWeak(Ptr);
       break;
     }
   }
 
   // When optimizing for size, call 'operator delete' unconditionally.
   if (CGF.CGM.getCodeGenOpts().OptimizeSize > 1) {
     CGF.EmitBlock(UnconditionalDeleteBlock);
     CGF.PopCleanupBlock();
     return true;
   }
 
   CGF.PopCleanupBlock();
   return false;
 }
 
 namespace {
   /// Calls the given 'operator delete' on an array of objects.
   struct CallArrayDelete final : EHScopeStack::Cleanup {
     llvm::Value *Ptr;
     const FunctionDecl *OperatorDelete;
     llvm::Value *NumElements;
     QualType ElementType;
     CharUnits CookieSize;
 
     CallArrayDelete(llvm::Value *Ptr,
                     const FunctionDecl *OperatorDelete,
                     llvm::Value *NumElements,
                     QualType ElementType,
                     CharUnits CookieSize)
       : Ptr(Ptr), OperatorDelete(OperatorDelete), NumElements(NumElements),
         ElementType(ElementType), CookieSize(CookieSize) {}
 
     void Emit(CodeGenFunction &CGF, Flags flags) override {
       CGF.EmitDeleteCall(OperatorDelete, Ptr, ElementType, NumElements,
                          CookieSize);
     }
   };
 }
 
 /// Emit the code for deleting an array of objects.
 static void EmitArrayDelete(CodeGenFunction &CGF,
                             const CXXDeleteExpr *E,
                             Address deletedPtr,
                             QualType elementType) {
   llvm::Value *numElements = nullptr;
   llvm::Value *allocatedPtr = nullptr;
   CharUnits cookieSize;
   CGF.CGM.getCXXABI().ReadArrayCookie(CGF, deletedPtr, E, elementType,
                                       numElements, allocatedPtr, cookieSize);
 
   assert(allocatedPtr && "ReadArrayCookie didn't set allocated pointer");
 
   // Make sure that we call delete even if one of the dtors throws.
   const FunctionDecl *operatorDelete = E->getOperatorDelete();
   CGF.EHStack.pushCleanup<CallArrayDelete>(NormalAndEHCleanup,
                                            allocatedPtr, operatorDelete,
                                            numElements, elementType,
                                            cookieSize);
 
   // Destroy the elements.
   if (QualType::DestructionKind dtorKind = elementType.isDestructedType()) {
     assert(numElements && "no element count for a type with a destructor!");
 
     CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);
     CharUnits elementAlign =
       deletedPtr.getAlignment().alignmentOfArrayElement(elementSize);
 
     llvm::Value *arrayBegin = deletedPtr.getPointer();
     llvm::Value *arrayEnd = CGF.Builder.CreateInBoundsGEP(
       deletedPtr.getElementType(), arrayBegin, numElements, "delete.end");
 
     // Note that it is legal to allocate a zero-length array, and we
     // can never fold the check away because the length should always
     // come from a cookie.
     CGF.emitArrayDestroy(arrayBegin, arrayEnd, elementType, elementAlign,
                          CGF.getDestroyer(dtorKind),
                          /*checkZeroLength*/ true,
                          CGF.needsEHCleanup(dtorKind));
   }
 
   // Pop the cleanup block.
   CGF.PopCleanupBlock();
 }
 
 void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {
   const Expr *Arg = E->getArgument();
   Address Ptr = EmitPointerWithAlignment(Arg);
 
   // Null check the pointer.
   //
   // We could avoid this null check if we can determine that the object
   // destruction is trivial and doesn't require an array cookie; we can
   // unconditionally perform the operator delete call in that case. For now, we
   // assume that deleted pointers are null rarely enough that it's better to
   // keep the branch. This might be worth revisiting for a -O0 code size win.
   llvm::BasicBlock *DeleteNotNull = createBasicBlock("delete.notnull");
   llvm::BasicBlock *DeleteEnd = createBasicBlock("delete.end");
 
   llvm::Value *IsNull = Builder.CreateIsNull(Ptr.getPointer(), "isnull");
 
   Builder.CreateCondBr(IsNull, DeleteEnd, DeleteNotNull);
   EmitBlock(DeleteNotNull);
 
   QualType DeleteTy = E->getDestroyedType();
 
   // A destroying operator delete overrides the entire operation of the
   // delete expression.
   if (E->getOperatorDelete()->isDestroyingOperatorDelete()) {
     EmitDestroyingObjectDelete(*this, E, Ptr, DeleteTy);
     EmitBlock(DeleteEnd);
     return;
   }
 
   // We might be deleting a pointer to array.  If so, GEP down to the
   // first non-array element.
   // (this assumes that A(*)[3][7] is converted to [3 x [7 x %A]]*)
   if (DeleteTy->isConstantArrayType()) {
     llvm::Value *Zero = Builder.getInt32(0);
     SmallVector<llvm::Value*,8> GEP;
 
     GEP.push_back(Zero); // point at the outermost array
 
     // For each layer of array type we're pointing at:
     while (const ConstantArrayType *Arr
              = getContext().getAsConstantArrayType(DeleteTy)) {
       // 1. Unpeel the array type.
       DeleteTy = Arr->getElementType();
 
       // 2. GEP to the first element of the array.
       GEP.push_back(Zero);
     }
 
     Ptr = Address(Builder.CreateInBoundsGEP(Ptr.getElementType(),
                                             Ptr.getPointer(), GEP, "del.first"),
                   Ptr.getAlignment());
   }
 
   assert(ConvertTypeForMem(DeleteTy) == Ptr.getElementType());
 
   if (E->isArrayForm()) {
     EmitArrayDelete(*this, E, Ptr, DeleteTy);
     EmitBlock(DeleteEnd);
   } else {
     if (!EmitObjectDelete(*this, E, Ptr, DeleteTy, DeleteEnd))
       EmitBlock(DeleteEnd);
   }
 }
 
 static bool isGLValueFromPointerDeref(const Expr *E) {
   E = E->IgnoreParens();
 
   if (const auto *CE = dyn_cast<CastExpr>(E)) {
     if (!CE->getSubExpr()->isGLValue())
       return false;
     return isGLValueFromPointerDeref(CE->getSubExpr());
   }
 
   if (const auto *OVE = dyn_cast<OpaqueValueExpr>(E))
     return isGLValueFromPointerDeref(OVE->getSourceExpr());
 
   if (const auto *BO = dyn_cast<BinaryOperator>(E))
     if (BO->getOpcode() == BO_Comma)
       return isGLValueFromPointerDeref(BO->getRHS());
 
   if (const auto *ACO = dyn_cast<AbstractConditionalOperator>(E))
     return isGLValueFromPointerDeref(ACO->getTrueExpr()) ||
            isGLValueFromPointerDeref(ACO->getFalseExpr());
 
   // C++11 [expr.sub]p1:
   //   The expression E1[E2] is identical (by definition) to *((E1)+(E2))
   if (isa<ArraySubscriptExpr>(E))
     return true;
 
   if (const auto *UO = dyn_cast<UnaryOperator>(E))
     if (UO->getOpcode() == UO_Deref)
       return true;
 
   return false;
 }
 
 static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E,
                                          llvm::Type *StdTypeInfoPtrTy) {
   // Get the vtable pointer.
   Address ThisPtr = CGF.EmitLValue(E).getAddress(CGF);
 
   QualType SrcRecordTy = E->getType();
 
   // C++ [class.cdtor]p4:
   //   If the operand of typeid refers to the object under construction or
   //   destruction and the static type of the operand is neither the constructor
   //   or destructor’s class nor one of its bases, the behavior is undefined.
   CGF.EmitTypeCheck(CodeGenFunction::TCK_DynamicOperation, E->getExprLoc(),
                     ThisPtr.getPointer(), SrcRecordTy);
 
   // C++ [expr.typeid]p2:
   //   If the glvalue expression is obtained by applying the unary * operator to
   //   a pointer and the pointer is a null pointer value, the typeid expression
   //   throws the std::bad_typeid exception.
   //
   // However, this paragraph's intent is not clear.  We choose a very generous
   // interpretation which implores us to consider comma operators, conditional
   // operators, parentheses and other such constructs.
   if (CGF.CGM.getCXXABI().shouldTypeidBeNullChecked(
           isGLValueFromPointerDeref(E), SrcRecordTy)) {
     llvm::BasicBlock *BadTypeidBlock =
         CGF.createBasicBlock("typeid.bad_typeid");
     llvm::BasicBlock *EndBlock = CGF.createBasicBlock("typeid.end");
 
     llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr.getPointer());
     CGF.Builder.CreateCondBr(IsNull, BadTypeidBlock, EndBlock);
 
     CGF.EmitBlock(BadTypeidBlock);
     CGF.CGM.getCXXABI().EmitBadTypeidCall(CGF);
     CGF.EmitBlock(EndBlock);
   }
 
   return CGF.CGM.getCXXABI().EmitTypeid(CGF, SrcRecordTy, ThisPtr,
                                         StdTypeInfoPtrTy);
 }
 
 llvm::Value *CodeGenFunction::EmitCXXTypeidExpr(const CXXTypeidExpr *E) {
   llvm::Type *StdTypeInfoPtrTy =
     ConvertType(E->getType())->getPointerTo();
 
   if (E->isTypeOperand()) {
     llvm::Constant *TypeInfo =
         CGM.GetAddrOfRTTIDescriptor(E->getTypeOperand(getContext()));
     return Builder.CreateBitCast(TypeInfo, StdTypeInfoPtrTy);
   }
 
   // C++ [expr.typeid]p2:
   //   When typeid is applied to a glvalue expression whose type is a
   //   polymorphic class type, the result refers to a std::type_info object
   //   representing the type of the most derived object (that is, the dynamic
   //   type) to which the glvalue refers.
   // If the operand is already most derived object, no need to look up vtable.
   if (E->isPotentiallyEvaluated() && !E->isMostDerived(getContext()))
     return EmitTypeidFromVTable(*this, E->getExprOperand(),
                                 StdTypeInfoPtrTy);
 
   QualType OperandTy = E->getExprOperand()->getType();
   return Builder.CreateBitCast(CGM.GetAddrOfRTTIDescriptor(OperandTy),
                                StdTypeInfoPtrTy);
 }
 
 static llvm::Value *EmitDynamicCastToNull(CodeGenFunction &CGF,
                                           QualType DestTy) {
   llvm::Type *DestLTy = CGF.ConvertType(DestTy);
   if (DestTy->isPointerType())
     return llvm::Constant::getNullValue(DestLTy);
 
   /// C++ [expr.dynamic.cast]p9:
   ///   A failed cast to reference type throws std::bad_cast
   if (!CGF.CGM.getCXXABI().EmitBadCastCall(CGF))
     return nullptr;
 
   CGF.EmitBlock(CGF.createBasicBlock("dynamic_cast.end"));
   return llvm::UndefValue::get(DestLTy);
 }
 
 llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr,
                                               const CXXDynamicCastExpr *DCE) {
   CGM.EmitExplicitCastExprType(DCE, this);
   QualType DestTy = DCE->getTypeAsWritten();
 
   QualType SrcTy = DCE->getSubExpr()->getType();
 
   // C++ [expr.dynamic.cast]p7:
   //   If T is "pointer to cv void," then the result is a pointer to the most
   //   derived object pointed to by v.
   const PointerType *DestPTy = DestTy->getAs<PointerType>();
 
   bool isDynamicCastToVoid;
   QualType SrcRecordTy;
   QualType DestRecordTy;
   if (DestPTy) {
     isDynamicCastToVoid = DestPTy->getPointeeType()->isVoidType();
     SrcRecordTy = SrcTy->castAs<PointerType>()->getPointeeType();
     DestRecordTy = DestPTy->getPointeeType();
   } else {
     isDynamicCastToVoid = false;
     SrcRecordTy = SrcTy;
     DestRecordTy = DestTy->castAs<ReferenceType>()->getPointeeType();
   }
 
   // C++ [class.cdtor]p5:
   //   If the operand of the dynamic_cast refers to the object under
   //   construction or destruction and the static type of the operand is not a
   //   pointer to or object of the constructor or destructor’s own class or one
   //   of its bases, the dynamic_cast results in undefined behavior.
   EmitTypeCheck(TCK_DynamicOperation, DCE->getExprLoc(), ThisAddr.getPointer(),
                 SrcRecordTy);
 
   if (DCE->isAlwaysNull())
     if (llvm::Value *T = EmitDynamicCastToNull(*this, DestTy))
       return T;
 
   assert(SrcRecordTy->isRecordType() && "source type must be a record type!");
 
   // C++ [expr.dynamic.cast]p4:
   //   If the value of v is a null pointer value in the pointer case, the result
   //   is the null pointer value of type T.
   bool ShouldNullCheckSrcValue =
       CGM.getCXXABI().shouldDynamicCastCallBeNullChecked(SrcTy->isPointerType(),
                                                          SrcRecordTy);
 
   llvm::BasicBlock *CastNull = nullptr;
   llvm::BasicBlock *CastNotNull = nullptr;
   llvm::BasicBlock *CastEnd = createBasicBlock("dynamic_cast.end");
 
   if (ShouldNullCheckSrcValue) {
     CastNull = createBasicBlock("dynamic_cast.null");
     CastNotNull = createBasicBlock("dynamic_cast.notnull");
 
     llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr.getPointer());
     Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
     EmitBlock(CastNotNull);
   }
 
   llvm::Value *Value;
   if (isDynamicCastToVoid) {
     Value = CGM.getCXXABI().EmitDynamicCastToVoid(*this, ThisAddr, SrcRecordTy,
                                                   DestTy);
   } else {
     assert(DestRecordTy->isRecordType() &&
            "destination type must be a record type!");
     Value = CGM.getCXXABI().EmitDynamicCastCall(*this, ThisAddr, SrcRecordTy,
                                                 DestTy, DestRecordTy, CastEnd);
     CastNotNull = Builder.GetInsertBlock();
   }
 
   if (ShouldNullCheckSrcValue) {
     EmitBranch(CastEnd);
 
     EmitBlock(CastNull);
     EmitBranch(CastEnd);
   }
 
   EmitBlock(CastEnd);
 
   if (ShouldNullCheckSrcValue) {
     llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);
     PHI->addIncoming(Value, CastNotNull);
     PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull);
 
     Value = PHI;
   }
 
   return Value;
 }
diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp b/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp
index 29806b65e984..2777fc22600d 100644
--- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.cpp
@@ -1,6651 +1,6658 @@
 //===--- CodeGenModule.cpp - Emit LLVM Code from ASTs for a Module --------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This coordinates the per-module state used while generating code.
 //
 //===----------------------------------------------------------------------===//
 
 #include "CodeGenModule.h"
 #include "CGBlocks.h"
 #include "CGCUDARuntime.h"
 #include "CGCXXABI.h"
 #include "CGCall.h"
 #include "CGDebugInfo.h"
 #include "CGObjCRuntime.h"
 #include "CGOpenCLRuntime.h"
 #include "CGOpenMPRuntime.h"
 #include "CGOpenMPRuntimeGPU.h"
 #include "CodeGenFunction.h"
 #include "CodeGenPGO.h"
 #include "ConstantEmitter.h"
 #include "CoverageMappingGen.h"
 #include "TargetInfo.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Mangle.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/StmtVisitor.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/CharInfo.h"
 #include "clang/Basic/CodeGenOptions.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/Module.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/Version.h"
 #include "clang/CodeGen/ConstantInitBuilder.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/ProfileSummary.h"
 #include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/X86TargetParser.h"
 
 using namespace clang;
 using namespace CodeGen;
 
 static llvm::cl::opt<bool> LimitedCoverage(
     "limited-coverage-experimental", llvm::cl::ZeroOrMore, llvm::cl::Hidden,
     llvm::cl::desc("Emit limited coverage mapping information (experimental)"),
     llvm::cl::init(false));
 
 static const char AnnotationSection[] = "llvm.metadata";
 
 static CGCXXABI *createCXXABI(CodeGenModule &CGM) {
   switch (CGM.getContext().getCXXABIKind()) {
   case TargetCXXABI::AppleARM64:
   case TargetCXXABI::Fuchsia:
   case TargetCXXABI::GenericAArch64:
   case TargetCXXABI::GenericARM:
   case TargetCXXABI::iOS:
   case TargetCXXABI::WatchOS:
   case TargetCXXABI::GenericMIPS:
   case TargetCXXABI::GenericItanium:
   case TargetCXXABI::WebAssembly:
   case TargetCXXABI::XL:
     return CreateItaniumCXXABI(CGM);
   case TargetCXXABI::Microsoft:
     return CreateMicrosoftCXXABI(CGM);
   }
 
   llvm_unreachable("invalid C++ ABI kind");
 }
 
 CodeGenModule::CodeGenModule(ASTContext &C, const HeaderSearchOptions &HSO,
                              const PreprocessorOptions &PPO,
                              const CodeGenOptions &CGO, llvm::Module &M,
                              DiagnosticsEngine &diags,
                              CoverageSourceInfo *CoverageInfo)
     : Context(C), LangOpts(C.getLangOpts()), HeaderSearchOpts(HSO),
       PreprocessorOpts(PPO), CodeGenOpts(CGO), TheModule(M), Diags(diags),
       Target(C.getTargetInfo()), ABI(createCXXABI(*this)),
       VMContext(M.getContext()), Types(*this), VTables(*this),
       SanitizerMD(new SanitizerMetadata(*this)) {
 
   // Initialize the type cache.
   llvm::LLVMContext &LLVMContext = M.getContext();
   VoidTy = llvm::Type::getVoidTy(LLVMContext);
   Int8Ty = llvm::Type::getInt8Ty(LLVMContext);
   Int16Ty = llvm::Type::getInt16Ty(LLVMContext);
   Int32Ty = llvm::Type::getInt32Ty(LLVMContext);
   Int64Ty = llvm::Type::getInt64Ty(LLVMContext);
   HalfTy = llvm::Type::getHalfTy(LLVMContext);
   BFloatTy = llvm::Type::getBFloatTy(LLVMContext);
   FloatTy = llvm::Type::getFloatTy(LLVMContext);
   DoubleTy = llvm::Type::getDoubleTy(LLVMContext);
   PointerWidthInBits = C.getTargetInfo().getPointerWidth(0);
   PointerAlignInBytes =
     C.toCharUnitsFromBits(C.getTargetInfo().getPointerAlign(0)).getQuantity();
   SizeSizeInBytes =
     C.toCharUnitsFromBits(C.getTargetInfo().getMaxPointerWidth()).getQuantity();
   IntAlignInBytes =
     C.toCharUnitsFromBits(C.getTargetInfo().getIntAlign()).getQuantity();
   CharTy =
     llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getCharWidth());
   IntTy = llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getIntWidth());
   IntPtrTy = llvm::IntegerType::get(LLVMContext,
     C.getTargetInfo().getMaxPointerWidth());
   Int8PtrTy = Int8Ty->getPointerTo(0);
   Int8PtrPtrTy = Int8PtrTy->getPointerTo(0);
   const llvm::DataLayout &DL = M.getDataLayout();
   AllocaInt8PtrTy = Int8Ty->getPointerTo(DL.getAllocaAddrSpace());
   GlobalsInt8PtrTy = Int8Ty->getPointerTo(DL.getDefaultGlobalsAddressSpace());
   ASTAllocaAddressSpace = getTargetCodeGenInfo().getASTAllocaAddressSpace();
 
   RuntimeCC = getTargetCodeGenInfo().getABIInfo().getRuntimeCC();
 
   if (LangOpts.ObjC)
     createObjCRuntime();
   if (LangOpts.OpenCL)
     createOpenCLRuntime();
   if (LangOpts.OpenMP)
     createOpenMPRuntime();
   if (LangOpts.CUDA)
     createCUDARuntime();
 
   // Enable TBAA unless it's suppressed. ThreadSanitizer needs TBAA even at O0.
   if (LangOpts.Sanitize.has(SanitizerKind::Thread) ||
       (!CodeGenOpts.RelaxedAliasing && CodeGenOpts.OptimizationLevel > 0))
     TBAA.reset(new CodeGenTBAA(Context, TheModule, CodeGenOpts, getLangOpts(),
                                getCXXABI().getMangleContext()));
 
   // If debug info or coverage generation is enabled, create the CGDebugInfo
   // object.
   if (CodeGenOpts.getDebugInfo() != codegenoptions::NoDebugInfo ||
       CodeGenOpts.EmitGcovArcs || CodeGenOpts.EmitGcovNotes)
     DebugInfo.reset(new CGDebugInfo(*this));
 
   Block.GlobalUniqueCount = 0;
 
   if (C.getLangOpts().ObjC)
     ObjCData.reset(new ObjCEntrypoints());
 
   if (CodeGenOpts.hasProfileClangUse()) {
     auto ReaderOrErr = llvm::IndexedInstrProfReader::create(
         CodeGenOpts.ProfileInstrumentUsePath, CodeGenOpts.ProfileRemappingFile);
     if (auto E = ReaderOrErr.takeError()) {
       unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
                                               "Could not read profile %0: %1");
       llvm::handleAllErrors(std::move(E), [&](const llvm::ErrorInfoBase &EI) {
         getDiags().Report(DiagID) << CodeGenOpts.ProfileInstrumentUsePath
                                   << EI.message();
       });
     } else
       PGOReader = std::move(ReaderOrErr.get());
   }
 
   // If coverage mapping generation is enabled, create the
   // CoverageMappingModuleGen object.
   if (CodeGenOpts.CoverageMapping)
     CoverageMapping.reset(new CoverageMappingModuleGen(*this, *CoverageInfo));
 
   // Generate the module name hash here if needed.
   if (CodeGenOpts.UniqueInternalLinkageNames &&
       !getModule().getSourceFileName().empty()) {
     std::string Path = getModule().getSourceFileName();
     // Check if a path substitution is needed from the MacroPrefixMap.
     for (const auto &Entry : LangOpts.MacroPrefixMap)
       if (Path.rfind(Entry.first, 0) != std::string::npos) {
         Path = Entry.second + Path.substr(Entry.first.size());
         break;
       }
     llvm::MD5 Md5;
     Md5.update(Path);
     llvm::MD5::MD5Result R;
     Md5.final(R);
     SmallString<32> Str;
     llvm::MD5::stringifyResult(R, Str);
     // Convert MD5hash to Decimal. Demangler suffixes can either contain
     // numbers or characters but not both.
     llvm::APInt IntHash(128, Str.str(), 16);
     // Prepend "__uniq" before the hash for tools like profilers to understand
     // that this symbol is of internal linkage type.  The "__uniq" is the
     // pre-determined prefix that is used to tell tools that this symbol was
     // created with -funique-internal-linakge-symbols and the tools can strip or
     // keep the prefix as needed.
     ModuleNameHash = (Twine(".__uniq.") +
         Twine(toString(IntHash, /* Radix = */ 10, /* Signed = */false))).str();
   }
 }
 
 CodeGenModule::~CodeGenModule() {}
 
 void CodeGenModule::createObjCRuntime() {
   // This is just isGNUFamily(), but we want to force implementors of
   // new ABIs to decide how best to do this.
   switch (LangOpts.ObjCRuntime.getKind()) {
   case ObjCRuntime::GNUstep:
   case ObjCRuntime::GCC:
   case ObjCRuntime::ObjFW:
     ObjCRuntime.reset(CreateGNUObjCRuntime(*this));
     return;
 
   case ObjCRuntime::FragileMacOSX:
   case ObjCRuntime::MacOSX:
   case ObjCRuntime::iOS:
   case ObjCRuntime::WatchOS:
     ObjCRuntime.reset(CreateMacObjCRuntime(*this));
     return;
   }
   llvm_unreachable("bad runtime kind");
 }
 
 void CodeGenModule::createOpenCLRuntime() {
   OpenCLRuntime.reset(new CGOpenCLRuntime(*this));
 }
 
 void CodeGenModule::createOpenMPRuntime() {
   // Select a specialized code generation class based on the target, if any.
   // If it does not exist use the default implementation.
   switch (getTriple().getArch()) {
   case llvm::Triple::nvptx:
   case llvm::Triple::nvptx64:
   case llvm::Triple::amdgcn:
     assert(getLangOpts().OpenMPIsDevice &&
            "OpenMP AMDGPU/NVPTX is only prepared to deal with device code.");
     OpenMPRuntime.reset(new CGOpenMPRuntimeGPU(*this));
     break;
   default:
     if (LangOpts.OpenMPSimd)
       OpenMPRuntime.reset(new CGOpenMPSIMDRuntime(*this));
     else
       OpenMPRuntime.reset(new CGOpenMPRuntime(*this));
     break;
   }
 }
 
 void CodeGenModule::createCUDARuntime() {
   CUDARuntime.reset(CreateNVCUDARuntime(*this));
 }
 
 void CodeGenModule::addReplacement(StringRef Name, llvm::Constant *C) {
   Replacements[Name] = C;
 }
 
 void CodeGenModule::applyReplacements() {
   for (auto &I : Replacements) {
     StringRef MangledName = I.first();
     llvm::Constant *Replacement = I.second;
     llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
     if (!Entry)
       continue;
     auto *OldF = cast<llvm::Function>(Entry);
     auto *NewF = dyn_cast<llvm::Function>(Replacement);
     if (!NewF) {
       if (auto *Alias = dyn_cast<llvm::GlobalAlias>(Replacement)) {
         NewF = dyn_cast<llvm::Function>(Alias->getAliasee());
       } else {
         auto *CE = cast<llvm::ConstantExpr>(Replacement);
         assert(CE->getOpcode() == llvm::Instruction::BitCast ||
                CE->getOpcode() == llvm::Instruction::GetElementPtr);
         NewF = dyn_cast<llvm::Function>(CE->getOperand(0));
       }
     }
 
     // Replace old with new, but keep the old order.
     OldF->replaceAllUsesWith(Replacement);
     if (NewF) {
       NewF->removeFromParent();
       OldF->getParent()->getFunctionList().insertAfter(OldF->getIterator(),
                                                        NewF);
     }
     OldF->eraseFromParent();
   }
 }
 
 void CodeGenModule::addGlobalValReplacement(llvm::GlobalValue *GV, llvm::Constant *C) {
   GlobalValReplacements.push_back(std::make_pair(GV, C));
 }
 
 void CodeGenModule::applyGlobalValReplacements() {
   for (auto &I : GlobalValReplacements) {
     llvm::GlobalValue *GV = I.first;
     llvm::Constant *C = I.second;
 
     GV->replaceAllUsesWith(C);
     GV->eraseFromParent();
   }
 }
 
 // This is only used in aliases that we created and we know they have a
 // linear structure.
 static const llvm::GlobalValue *getAliasedGlobal(const llvm::GlobalValue *GV) {
   const llvm::Constant *C;
   if (auto *GA = dyn_cast<llvm::GlobalAlias>(GV))
     C = GA->getAliasee();
   else if (auto *GI = dyn_cast<llvm::GlobalIFunc>(GV))
     C = GI->getResolver();
   else
     return GV;
 
   const auto *AliaseeGV = dyn_cast<llvm::GlobalValue>(C->stripPointerCasts());
   if (!AliaseeGV)
     return nullptr;
 
   const llvm::GlobalValue *FinalGV = AliaseeGV->getAliaseeObject();
   if (FinalGV == GV)
     return nullptr;
 
   return FinalGV;
 }
 
 static bool checkAliasedGlobal(DiagnosticsEngine &Diags,
                                SourceLocation Location, bool IsIFunc,
                                const llvm::GlobalValue *Alias,
                                const llvm::GlobalValue *&GV) {
   GV = getAliasedGlobal(Alias);
   if (!GV) {
     Diags.Report(Location, diag::err_cyclic_alias) << IsIFunc;
     return false;
   }
 
   if (GV->isDeclaration()) {
     Diags.Report(Location, diag::err_alias_to_undefined) << IsIFunc << IsIFunc;
     return false;
   }
 
   if (IsIFunc) {
     // Check resolver function type.
     const auto *F = dyn_cast<llvm::Function>(GV);
     if (!F) {
       Diags.Report(Location, diag::err_alias_to_undefined)
           << IsIFunc << IsIFunc;
       return false;
     }
 
     llvm::FunctionType *FTy = F->getFunctionType();
     if (!FTy->getReturnType()->isPointerTy()) {
       Diags.Report(Location, diag::err_ifunc_resolver_return);
       return false;
     }
   }
 
   return true;
 }
 
 void CodeGenModule::checkAliases() {
   // Check if the constructed aliases are well formed. It is really unfortunate
   // that we have to do this in CodeGen, but we only construct mangled names
   // and aliases during codegen.
   bool Error = false;
   DiagnosticsEngine &Diags = getDiags();
   for (const GlobalDecl &GD : Aliases) {
     const auto *D = cast<ValueDecl>(GD.getDecl());
     SourceLocation Location;
     bool IsIFunc = D->hasAttr<IFuncAttr>();
     if (const Attr *A = D->getDefiningAttr())
       Location = A->getLocation();
     else
       llvm_unreachable("Not an alias or ifunc?");
 
     StringRef MangledName = getMangledName(GD);
     llvm::GlobalValue *Alias = GetGlobalValue(MangledName);
     const llvm::GlobalValue *GV = nullptr;
     if (!checkAliasedGlobal(Diags, Location, IsIFunc, Alias, GV)) {
       Error = true;
       continue;
     }
 
     llvm::Constant *Aliasee =
         IsIFunc ? cast<llvm::GlobalIFunc>(Alias)->getResolver()
                 : cast<llvm::GlobalAlias>(Alias)->getAliasee();
 
     llvm::GlobalValue *AliaseeGV;
     if (auto CE = dyn_cast<llvm::ConstantExpr>(Aliasee))
       AliaseeGV = cast<llvm::GlobalValue>(CE->getOperand(0));
     else
       AliaseeGV = cast<llvm::GlobalValue>(Aliasee);
 
     if (const SectionAttr *SA = D->getAttr<SectionAttr>()) {
       StringRef AliasSection = SA->getName();
       if (AliasSection != AliaseeGV->getSection())
         Diags.Report(SA->getLocation(), diag::warn_alias_with_section)
             << AliasSection << IsIFunc << IsIFunc;
     }
 
     // We have to handle alias to weak aliases in here. LLVM itself disallows
     // this since the object semantics would not match the IL one. For
     // compatibility with gcc we implement it by just pointing the alias
     // to its aliasee's aliasee. We also warn, since the user is probably
     // expecting the link to be weak.
     if (auto *GA = dyn_cast<llvm::GlobalAlias>(AliaseeGV)) {
       if (GA->isInterposable()) {
         Diags.Report(Location, diag::warn_alias_to_weak_alias)
             << GV->getName() << GA->getName() << IsIFunc;
         Aliasee = llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
             GA->getAliasee(), Alias->getType());
 
         if (IsIFunc)
           cast<llvm::GlobalIFunc>(Alias)->setResolver(Aliasee);
         else
           cast<llvm::GlobalAlias>(Alias)->setAliasee(Aliasee);
       }
     }
   }
   if (!Error)
     return;
 
   for (const GlobalDecl &GD : Aliases) {
     StringRef MangledName = getMangledName(GD);
     llvm::GlobalValue *Alias = GetGlobalValue(MangledName);
     Alias->replaceAllUsesWith(llvm::UndefValue::get(Alias->getType()));
     Alias->eraseFromParent();
   }
 }
 
 void CodeGenModule::clear() {
   DeferredDeclsToEmit.clear();
   if (OpenMPRuntime)
     OpenMPRuntime->clear();
 }
 
 void InstrProfStats::reportDiagnostics(DiagnosticsEngine &Diags,
                                        StringRef MainFile) {
   if (!hasDiagnostics())
     return;
   if (VisitedInMainFile > 0 && VisitedInMainFile == MissingInMainFile) {
     if (MainFile.empty())
       MainFile = "<stdin>";
     Diags.Report(diag::warn_profile_data_unprofiled) << MainFile;
   } else {
     if (Mismatched > 0)
       Diags.Report(diag::warn_profile_data_out_of_date) << Visited << Mismatched;
 
     if (Missing > 0)
       Diags.Report(diag::warn_profile_data_missing) << Visited << Missing;
   }
 }
 
 static void setVisibilityFromDLLStorageClass(const clang::LangOptions &LO,
                                              llvm::Module &M) {
   if (!LO.VisibilityFromDLLStorageClass)
     return;
 
   llvm::GlobalValue::VisibilityTypes DLLExportVisibility =
       CodeGenModule::GetLLVMVisibility(LO.getDLLExportVisibility());
   llvm::GlobalValue::VisibilityTypes NoDLLStorageClassVisibility =
       CodeGenModule::GetLLVMVisibility(LO.getNoDLLStorageClassVisibility());
   llvm::GlobalValue::VisibilityTypes ExternDeclDLLImportVisibility =
       CodeGenModule::GetLLVMVisibility(LO.getExternDeclDLLImportVisibility());
   llvm::GlobalValue::VisibilityTypes ExternDeclNoDLLStorageClassVisibility =
       CodeGenModule::GetLLVMVisibility(
           LO.getExternDeclNoDLLStorageClassVisibility());
 
   for (llvm::GlobalValue &GV : M.global_values()) {
     if (GV.hasAppendingLinkage() || GV.hasLocalLinkage())
       continue;
 
     // Reset DSO locality before setting the visibility. This removes
     // any effects that visibility options and annotations may have
     // had on the DSO locality. Setting the visibility will implicitly set
     // appropriate globals to DSO Local; however, this will be pessimistic
     // w.r.t. to the normal compiler IRGen.
     GV.setDSOLocal(false);
 
     if (GV.isDeclarationForLinker()) {
       GV.setVisibility(GV.getDLLStorageClass() ==
                                llvm::GlobalValue::DLLImportStorageClass
                            ? ExternDeclDLLImportVisibility
                            : ExternDeclNoDLLStorageClassVisibility);
     } else {
       GV.setVisibility(GV.getDLLStorageClass() ==
                                llvm::GlobalValue::DLLExportStorageClass
                            ? DLLExportVisibility
                            : NoDLLStorageClassVisibility);
     }
 
     GV.setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
   }
 }
 
 void CodeGenModule::Release() {
   EmitDeferred();
   EmitVTablesOpportunistically();
   applyGlobalValReplacements();
   applyReplacements();
   checkAliases();
   emitMultiVersionFunctions();
   EmitCXXGlobalInitFunc();
   EmitCXXGlobalCleanUpFunc();
   registerGlobalDtorsWithAtExit();
   EmitCXXThreadLocalInitFunc();
   if (ObjCRuntime)
     if (llvm::Function *ObjCInitFunction = ObjCRuntime->ModuleInitFunction())
       AddGlobalCtor(ObjCInitFunction);
   if (Context.getLangOpts().CUDA && CUDARuntime) {
     if (llvm::Function *CudaCtorFunction = CUDARuntime->finalizeModule())
       AddGlobalCtor(CudaCtorFunction);
   }
   if (OpenMPRuntime) {
     if (llvm::Function *OpenMPRequiresDirectiveRegFun =
             OpenMPRuntime->emitRequiresDirectiveRegFun()) {
       AddGlobalCtor(OpenMPRequiresDirectiveRegFun, 0);
     }
     OpenMPRuntime->createOffloadEntriesAndInfoMetadata();
     OpenMPRuntime->clear();
   }
   if (PGOReader) {
     getModule().setProfileSummary(
         PGOReader->getSummary(/* UseCS */ false).getMD(VMContext),
         llvm::ProfileSummary::PSK_Instr);
     if (PGOStats.hasDiagnostics())
       PGOStats.reportDiagnostics(getDiags(), getCodeGenOpts().MainFileName);
   }
   EmitCtorList(GlobalCtors, "llvm.global_ctors");
   EmitCtorList(GlobalDtors, "llvm.global_dtors");
   EmitGlobalAnnotations();
   EmitStaticExternCAliases();
   EmitDeferredUnusedCoverageMappings();
   CodeGenPGO(*this).setValueProfilingFlag(getModule());
   if (CoverageMapping)
     CoverageMapping->emit();
   if (CodeGenOpts.SanitizeCfiCrossDso) {
     CodeGenFunction(*this).EmitCfiCheckFail();
     CodeGenFunction(*this).EmitCfiCheckStub();
   }
   emitAtAvailableLinkGuard();
   if (Context.getTargetInfo().getTriple().isWasm() &&
       !Context.getTargetInfo().getTriple().isOSEmscripten()) {
     EmitMainVoidAlias();
   }
 
   // Emit reference of __amdgpu_device_library_preserve_asan_functions to
   // preserve ASAN functions in bitcode libraries.
   if (LangOpts.Sanitize.has(SanitizerKind::Address) && getTriple().isAMDGPU()) {
     auto *FT = llvm::FunctionType::get(VoidTy, {});
     auto *F = llvm::Function::Create(
         FT, llvm::GlobalValue::ExternalLinkage,
         "__amdgpu_device_library_preserve_asan_functions", &getModule());
     auto *Var = new llvm::GlobalVariable(
         getModule(), FT->getPointerTo(),
         /*isConstant=*/true, llvm::GlobalValue::WeakAnyLinkage, F,
         "__amdgpu_device_library_preserve_asan_functions_ptr", nullptr,
         llvm::GlobalVariable::NotThreadLocal);
     addCompilerUsedGlobal(Var);
     if (!getModule().getModuleFlag("amdgpu_hostcall")) {
       getModule().addModuleFlag(llvm::Module::Override, "amdgpu_hostcall", 1);
     }
   }
 
   emitLLVMUsed();
   if (SanStats)
     SanStats->finish();
 
   if (CodeGenOpts.Autolink &&
       (Context.getLangOpts().Modules || !LinkerOptionsMetadata.empty())) {
     EmitModuleLinkOptions();
   }
 
   // On ELF we pass the dependent library specifiers directly to the linker
   // without manipulating them. This is in contrast to other platforms where
   // they are mapped to a specific linker option by the compiler. This
   // difference is a result of the greater variety of ELF linkers and the fact
   // that ELF linkers tend to handle libraries in a more complicated fashion
   // than on other platforms. This forces us to defer handling the dependent
   // libs to the linker.
   //
   // CUDA/HIP device and host libraries are different. Currently there is no
   // way to differentiate dependent libraries for host or device. Existing
   // usage of #pragma comment(lib, *) is intended for host libraries on
   // Windows. Therefore emit llvm.dependent-libraries only for host.
   if (!ELFDependentLibraries.empty() && !Context.getLangOpts().CUDAIsDevice) {
     auto *NMD = getModule().getOrInsertNamedMetadata("llvm.dependent-libraries");
     for (auto *MD : ELFDependentLibraries)
       NMD->addOperand(MD);
   }
 
   // Record mregparm value now so it is visible through rest of codegen.
   if (Context.getTargetInfo().getTriple().getArch() == llvm::Triple::x86)
     getModule().addModuleFlag(llvm::Module::Error, "NumRegisterParameters",
                               CodeGenOpts.NumRegisterParameters);
 
   if (CodeGenOpts.DwarfVersion) {
     getModule().addModuleFlag(llvm::Module::Max, "Dwarf Version",
                               CodeGenOpts.DwarfVersion);
   }
 
   if (CodeGenOpts.Dwarf64)
     getModule().addModuleFlag(llvm::Module::Max, "DWARF64", 1);
 
   if (Context.getLangOpts().SemanticInterposition)
     // Require various optimization to respect semantic interposition.
     getModule().setSemanticInterposition(true);
 
   if (CodeGenOpts.EmitCodeView) {
     // Indicate that we want CodeView in the metadata.
     getModule().addModuleFlag(llvm::Module::Warning, "CodeView", 1);
   }
   if (CodeGenOpts.CodeViewGHash) {
     getModule().addModuleFlag(llvm::Module::Warning, "CodeViewGHash", 1);
   }
   if (CodeGenOpts.ControlFlowGuard) {
     // Function ID tables and checks for Control Flow Guard (cfguard=2).
     getModule().addModuleFlag(llvm::Module::Warning, "cfguard", 2);
   } else if (CodeGenOpts.ControlFlowGuardNoChecks) {
     // Function ID tables for Control Flow Guard (cfguard=1).
     getModule().addModuleFlag(llvm::Module::Warning, "cfguard", 1);
   }
   if (CodeGenOpts.EHContGuard) {
     // Function ID tables for EH Continuation Guard.
     getModule().addModuleFlag(llvm::Module::Warning, "ehcontguard", 1);
   }
   if (CodeGenOpts.OptimizationLevel > 0 && CodeGenOpts.StrictVTablePointers) {
     // We don't support LTO with 2 with different StrictVTablePointers
     // FIXME: we could support it by stripping all the information introduced
     // by StrictVTablePointers.
 
     getModule().addModuleFlag(llvm::Module::Error, "StrictVTablePointers",1);
 
     llvm::Metadata *Ops[2] = {
               llvm::MDString::get(VMContext, "StrictVTablePointers"),
               llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
                   llvm::Type::getInt32Ty(VMContext), 1))};
 
     getModule().addModuleFlag(llvm::Module::Require,
                               "StrictVTablePointersRequirement",
                               llvm::MDNode::get(VMContext, Ops));
   }
   if (getModuleDebugInfo())
     // We support a single version in the linked module. The LLVM
     // parser will drop debug info with a different version number
     // (and warn about it, too).
     getModule().addModuleFlag(llvm::Module::Warning, "Debug Info Version",
                               llvm::DEBUG_METADATA_VERSION);
 
   // We need to record the widths of enums and wchar_t, so that we can generate
   // the correct build attributes in the ARM backend. wchar_size is also used by
   // TargetLibraryInfo.
   uint64_t WCharWidth =
       Context.getTypeSizeInChars(Context.getWideCharType()).getQuantity();
   getModule().addModuleFlag(llvm::Module::Error, "wchar_size", WCharWidth);
 
   llvm::Triple::ArchType Arch = Context.getTargetInfo().getTriple().getArch();
   if (   Arch == llvm::Triple::arm
       || Arch == llvm::Triple::armeb
       || Arch == llvm::Triple::thumb
       || Arch == llvm::Triple::thumbeb) {
     // The minimum width of an enum in bytes
     uint64_t EnumWidth = Context.getLangOpts().ShortEnums ? 1 : 4;
     getModule().addModuleFlag(llvm::Module::Error, "min_enum_size", EnumWidth);
   }
 
   if (Arch == llvm::Triple::riscv32 || Arch == llvm::Triple::riscv64) {
     StringRef ABIStr = Target.getABI();
     llvm::LLVMContext &Ctx = TheModule.getContext();
     getModule().addModuleFlag(llvm::Module::Error, "target-abi",
                               llvm::MDString::get(Ctx, ABIStr));
   }
 
   if (CodeGenOpts.SanitizeCfiCrossDso) {
     // Indicate that we want cross-DSO control flow integrity checks.
     getModule().addModuleFlag(llvm::Module::Override, "Cross-DSO CFI", 1);
   }
 
   if (CodeGenOpts.WholeProgramVTables) {
     // Indicate whether VFE was enabled for this module, so that the
     // vcall_visibility metadata added under whole program vtables is handled
     // appropriately in the optimizer.
     getModule().addModuleFlag(llvm::Module::Error, "Virtual Function Elim",
                               CodeGenOpts.VirtualFunctionElimination);
   }
 
   if (LangOpts.Sanitize.has(SanitizerKind::CFIICall)) {
     getModule().addModuleFlag(llvm::Module::Override,
                               "CFI Canonical Jump Tables",
                               CodeGenOpts.SanitizeCfiCanonicalJumpTables);
   }
 
   if (CodeGenOpts.CFProtectionReturn &&
       Target.checkCFProtectionReturnSupported(getDiags())) {
     // Indicate that we want to instrument return control flow protection.
     getModule().addModuleFlag(llvm::Module::Override, "cf-protection-return",
                               1);
   }
 
   if (CodeGenOpts.CFProtectionBranch &&
       Target.checkCFProtectionBranchSupported(getDiags())) {
     // Indicate that we want to instrument branch control flow protection.
     getModule().addModuleFlag(llvm::Module::Override, "cf-protection-branch",
                               1);
   }
 
   if (CodeGenOpts.IBTSeal)
     getModule().addModuleFlag(llvm::Module::Override, "ibt-seal", 1);
 
   // Add module metadata for return address signing (ignoring
   // non-leaf/all) and stack tagging. These are actually turned on by function
   // attributes, but we use module metadata to emit build attributes. This is
   // needed for LTO, where the function attributes are inside bitcode
   // serialised into a global variable by the time build attributes are
   // emitted, so we can't access them.
   if (Context.getTargetInfo().hasFeature("ptrauth") &&
       LangOpts.getSignReturnAddressScope() !=
           LangOptions::SignReturnAddressScopeKind::None)
     getModule().addModuleFlag(llvm::Module::Override,
                               "sign-return-address-buildattr", 1);
   if (LangOpts.Sanitize.has(SanitizerKind::MemTag))
     getModule().addModuleFlag(llvm::Module::Override,
                               "tag-stack-memory-buildattr", 1);
 
   if (Arch == llvm::Triple::thumb || Arch == llvm::Triple::thumbeb ||
       Arch == llvm::Triple::arm || Arch == llvm::Triple::armeb ||
       Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_32 ||
       Arch == llvm::Triple::aarch64_be) {
     getModule().addModuleFlag(llvm::Module::Error, "branch-target-enforcement",
                               LangOpts.BranchTargetEnforcement);
 
     getModule().addModuleFlag(llvm::Module::Error, "sign-return-address",
                               LangOpts.hasSignReturnAddress());
 
     getModule().addModuleFlag(llvm::Module::Error, "sign-return-address-all",
                               LangOpts.isSignReturnAddressScopeAll());
 
     getModule().addModuleFlag(llvm::Module::Error,
                               "sign-return-address-with-bkey",
                               !LangOpts.isSignReturnAddressWithAKey());
   }
 
   if (!CodeGenOpts.MemoryProfileOutput.empty()) {
     llvm::LLVMContext &Ctx = TheModule.getContext();
     getModule().addModuleFlag(
         llvm::Module::Error, "MemProfProfileFilename",
         llvm::MDString::get(Ctx, CodeGenOpts.MemoryProfileOutput));
   }
 
   if (LangOpts.CUDAIsDevice && getTriple().isNVPTX()) {
     // Indicate whether __nvvm_reflect should be configured to flush denormal
     // floating point values to 0.  (This corresponds to its "__CUDA_FTZ"
     // property.)
     getModule().addModuleFlag(llvm::Module::Override, "nvvm-reflect-ftz",
                               CodeGenOpts.FP32DenormalMode.Output !=
                                   llvm::DenormalMode::IEEE);
   }
 
   if (LangOpts.EHAsynch)
     getModule().addModuleFlag(llvm::Module::Warning, "eh-asynch", 1);
 
   // Indicate whether this Module was compiled with -fopenmp
   if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd)
     getModule().addModuleFlag(llvm::Module::Max, "openmp", LangOpts.OpenMP);
   if (getLangOpts().OpenMPIsDevice)
     getModule().addModuleFlag(llvm::Module::Max, "openmp-device",
                               LangOpts.OpenMP);
 
   // Emit OpenCL specific module metadata: OpenCL/SPIR version.
   if (LangOpts.OpenCL) {
     EmitOpenCLMetadata();
     // Emit SPIR version.
     if (getTriple().isSPIR()) {
       // SPIR v2.0 s2.12 - The SPIR version used by the module is stored in the
       // opencl.spir.version named metadata.
       // C++ for OpenCL has a distinct mapping for version compatibility with
       // OpenCL.
       auto Version = LangOpts.getOpenCLCompatibleVersion();
       llvm::Metadata *SPIRVerElts[] = {
           llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
               Int32Ty, Version / 100)),
           llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
               Int32Ty, (Version / 100 > 1) ? 0 : 2))};
       llvm::NamedMDNode *SPIRVerMD =
           TheModule.getOrInsertNamedMetadata("opencl.spir.version");
       llvm::LLVMContext &Ctx = TheModule.getContext();
       SPIRVerMD->addOperand(llvm::MDNode::get(Ctx, SPIRVerElts));
     }
   }
 
   if (uint32_t PLevel = Context.getLangOpts().PICLevel) {
     assert(PLevel < 3 && "Invalid PIC Level");
     getModule().setPICLevel(static_cast<llvm::PICLevel::Level>(PLevel));
     if (Context.getLangOpts().PIE)
       getModule().setPIELevel(static_cast<llvm::PIELevel::Level>(PLevel));
   }
 
   if (getCodeGenOpts().CodeModel.size() > 0) {
     unsigned CM = llvm::StringSwitch<unsigned>(getCodeGenOpts().CodeModel)
                   .Case("tiny", llvm::CodeModel::Tiny)
                   .Case("small", llvm::CodeModel::Small)
                   .Case("kernel", llvm::CodeModel::Kernel)
                   .Case("medium", llvm::CodeModel::Medium)
                   .Case("large", llvm::CodeModel::Large)
                   .Default(~0u);
     if (CM != ~0u) {
       llvm::CodeModel::Model codeModel = static_cast<llvm::CodeModel::Model>(CM);
       getModule().setCodeModel(codeModel);
     }
   }
 
   if (CodeGenOpts.NoPLT)
     getModule().setRtLibUseGOT();
   if (CodeGenOpts.UnwindTables)
     getModule().setUwtable();
 
   switch (CodeGenOpts.getFramePointer()) {
   case CodeGenOptions::FramePointerKind::None:
     // 0 ("none") is the default.
     break;
   case CodeGenOptions::FramePointerKind::NonLeaf:
     getModule().setFramePointer(llvm::FramePointerKind::NonLeaf);
     break;
   case CodeGenOptions::FramePointerKind::All:
     getModule().setFramePointer(llvm::FramePointerKind::All);
     break;
   }
 
   SimplifyPersonality();
 
   if (getCodeGenOpts().EmitDeclMetadata)
     EmitDeclMetadata();
 
   if (getCodeGenOpts().EmitGcovArcs || getCodeGenOpts().EmitGcovNotes)
     EmitCoverageFile();
 
   if (CGDebugInfo *DI = getModuleDebugInfo())
     DI->finalize();
 
   if (getCodeGenOpts().EmitVersionIdentMetadata)
     EmitVersionIdentMetadata();
 
   if (!getCodeGenOpts().RecordCommandLine.empty())
     EmitCommandLineMetadata();
 
   if (!getCodeGenOpts().StackProtectorGuard.empty())
     getModule().setStackProtectorGuard(getCodeGenOpts().StackProtectorGuard);
   if (!getCodeGenOpts().StackProtectorGuardReg.empty())
     getModule().setStackProtectorGuardReg(
         getCodeGenOpts().StackProtectorGuardReg);
   if (getCodeGenOpts().StackProtectorGuardOffset != INT_MAX)
     getModule().setStackProtectorGuardOffset(
         getCodeGenOpts().StackProtectorGuardOffset);
   if (getCodeGenOpts().StackAlignment)
     getModule().setOverrideStackAlignment(getCodeGenOpts().StackAlignment);
   if (getCodeGenOpts().SkipRaxSetup)
     getModule().addModuleFlag(llvm::Module::Override, "SkipRaxSetup", 1);
 
   getTargetCodeGenInfo().emitTargetMetadata(*this, MangledDeclNames);
 
   EmitBackendOptionsMetadata(getCodeGenOpts());
 
   // Set visibility from DLL storage class
   // We do this at the end of LLVM IR generation; after any operation
   // that might affect the DLL storage class or the visibility, and
   // before anything that might act on these.
   setVisibilityFromDLLStorageClass(LangOpts, getModule());
 }
 
 void CodeGenModule::EmitOpenCLMetadata() {
   // SPIR v2.0 s2.13 - The OpenCL version used by the module is stored in the
   // opencl.ocl.version named metadata node.
   // C++ for OpenCL has a distinct mapping for versions compatibile with OpenCL.
   auto Version = LangOpts.getOpenCLCompatibleVersion();
   llvm::Metadata *OCLVerElts[] = {
       llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
           Int32Ty, Version / 100)),
       llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
           Int32Ty, (Version % 100) / 10))};
   llvm::NamedMDNode *OCLVerMD =
       TheModule.getOrInsertNamedMetadata("opencl.ocl.version");
   llvm::LLVMContext &Ctx = TheModule.getContext();
   OCLVerMD->addOperand(llvm::MDNode::get(Ctx, OCLVerElts));
 }
 
 void CodeGenModule::EmitBackendOptionsMetadata(
     const CodeGenOptions CodeGenOpts) {
   switch (getTriple().getArch()) {
   default:
     break;
   case llvm::Triple::riscv32:
   case llvm::Triple::riscv64:
     getModule().addModuleFlag(llvm::Module::Error, "SmallDataLimit",
                               CodeGenOpts.SmallDataLimit);
     break;
   }
 }
 
 void CodeGenModule::UpdateCompletedType(const TagDecl *TD) {
   // Make sure that this type is translated.
   Types.UpdateCompletedType(TD);
 }
 
 void CodeGenModule::RefreshTypeCacheForClass(const CXXRecordDecl *RD) {
   // Make sure that this type is translated.
   Types.RefreshTypeCacheForClass(RD);
 }
 
 llvm::MDNode *CodeGenModule::getTBAATypeInfo(QualType QTy) {
   if (!TBAA)
     return nullptr;
   return TBAA->getTypeInfo(QTy);
 }
 
 TBAAAccessInfo CodeGenModule::getTBAAAccessInfo(QualType AccessType) {
   if (!TBAA)
     return TBAAAccessInfo();
   if (getLangOpts().CUDAIsDevice) {
     // As CUDA builtin surface/texture types are replaced, skip generating TBAA
     // access info.
     if (AccessType->isCUDADeviceBuiltinSurfaceType()) {
       if (getTargetCodeGenInfo().getCUDADeviceBuiltinSurfaceDeviceType() !=
           nullptr)
         return TBAAAccessInfo();
     } else if (AccessType->isCUDADeviceBuiltinTextureType()) {
       if (getTargetCodeGenInfo().getCUDADeviceBuiltinTextureDeviceType() !=
           nullptr)
         return TBAAAccessInfo();
     }
   }
   return TBAA->getAccessInfo(AccessType);
 }
 
 TBAAAccessInfo
 CodeGenModule::getTBAAVTablePtrAccessInfo(llvm::Type *VTablePtrType) {
   if (!TBAA)
     return TBAAAccessInfo();
   return TBAA->getVTablePtrAccessInfo(VTablePtrType);
 }
 
 llvm::MDNode *CodeGenModule::getTBAAStructInfo(QualType QTy) {
   if (!TBAA)
     return nullptr;
   return TBAA->getTBAAStructInfo(QTy);
 }
 
 llvm::MDNode *CodeGenModule::getTBAABaseTypeInfo(QualType QTy) {
   if (!TBAA)
     return nullptr;
   return TBAA->getBaseTypeInfo(QTy);
 }
 
 llvm::MDNode *CodeGenModule::getTBAAAccessTagInfo(TBAAAccessInfo Info) {
   if (!TBAA)
     return nullptr;
   return TBAA->getAccessTagInfo(Info);
 }
 
 TBAAAccessInfo CodeGenModule::mergeTBAAInfoForCast(TBAAAccessInfo SourceInfo,
                                                    TBAAAccessInfo TargetInfo) {
   if (!TBAA)
     return TBAAAccessInfo();
   return TBAA->mergeTBAAInfoForCast(SourceInfo, TargetInfo);
 }
 
 TBAAAccessInfo
 CodeGenModule::mergeTBAAInfoForConditionalOperator(TBAAAccessInfo InfoA,
                                                    TBAAAccessInfo InfoB) {
   if (!TBAA)
     return TBAAAccessInfo();
   return TBAA->mergeTBAAInfoForConditionalOperator(InfoA, InfoB);
 }
 
 TBAAAccessInfo
 CodeGenModule::mergeTBAAInfoForMemoryTransfer(TBAAAccessInfo DestInfo,
                                               TBAAAccessInfo SrcInfo) {
   if (!TBAA)
     return TBAAAccessInfo();
   return TBAA->mergeTBAAInfoForConditionalOperator(DestInfo, SrcInfo);
 }
 
 void CodeGenModule::DecorateInstructionWithTBAA(llvm::Instruction *Inst,
                                                 TBAAAccessInfo TBAAInfo) {
   if (llvm::MDNode *Tag = getTBAAAccessTagInfo(TBAAInfo))
     Inst->setMetadata(llvm::LLVMContext::MD_tbaa, Tag);
 }
 
 void CodeGenModule::DecorateInstructionWithInvariantGroup(
     llvm::Instruction *I, const CXXRecordDecl *RD) {
   I->setMetadata(llvm::LLVMContext::MD_invariant_group,
                  llvm::MDNode::get(getLLVMContext(), {}));
 }
 
 void CodeGenModule::Error(SourceLocation loc, StringRef message) {
   unsigned diagID = getDiags().getCustomDiagID(DiagnosticsEngine::Error, "%0");
   getDiags().Report(Context.getFullLoc(loc), diagID) << message;
 }
 
 /// ErrorUnsupported - Print out an error that codegen doesn't support the
 /// specified stmt yet.
 void CodeGenModule::ErrorUnsupported(const Stmt *S, const char *Type) {
   unsigned DiagID = getDiags().getCustomDiagID(DiagnosticsEngine::Error,
                                                "cannot compile this %0 yet");
   std::string Msg = Type;
   getDiags().Report(Context.getFullLoc(S->getBeginLoc()), DiagID)
       << Msg << S->getSourceRange();
 }
 
 /// ErrorUnsupported - Print out an error that codegen doesn't support the
 /// specified decl yet.
 void CodeGenModule::ErrorUnsupported(const Decl *D, const char *Type) {
   unsigned DiagID = getDiags().getCustomDiagID(DiagnosticsEngine::Error,
                                                "cannot compile this %0 yet");
   std::string Msg = Type;
   getDiags().Report(Context.getFullLoc(D->getLocation()), DiagID) << Msg;
 }
 
 llvm::ConstantInt *CodeGenModule::getSize(CharUnits size) {
   return llvm::ConstantInt::get(SizeTy, size.getQuantity());
 }
 
 void CodeGenModule::setGlobalVisibility(llvm::GlobalValue *GV,
                                         const NamedDecl *D) const {
   if (GV->hasDLLImportStorageClass())
     return;
   // Internal definitions always have default visibility.
   if (GV->hasLocalLinkage()) {
     GV->setVisibility(llvm::GlobalValue::DefaultVisibility);
     return;
   }
   if (!D)
     return;
   // Set visibility for definitions, and for declarations if requested globally
   // or set explicitly.
   LinkageInfo LV = D->getLinkageAndVisibility();
   if (LV.isVisibilityExplicit() || getLangOpts().SetVisibilityForExternDecls ||
       !GV->isDeclarationForLinker())
     GV->setVisibility(GetLLVMVisibility(LV.getVisibility()));
 }
 
 static bool shouldAssumeDSOLocal(const CodeGenModule &CGM,
                                  llvm::GlobalValue *GV) {
   if (GV->hasLocalLinkage())
     return true;
 
   if (!GV->hasDefaultVisibility() && !GV->hasExternalWeakLinkage())
     return true;
 
   // DLLImport explicitly marks the GV as external.
   if (GV->hasDLLImportStorageClass())
     return false;
 
   const llvm::Triple &TT = CGM.getTriple();
   if (TT.isWindowsGNUEnvironment()) {
     // In MinGW, variables without DLLImport can still be automatically
     // imported from a DLL by the linker; don't mark variables that
     // potentially could come from another DLL as DSO local.
 
     // With EmulatedTLS, TLS variables can be autoimported from other DLLs
     // (and this actually happens in the public interface of libstdc++), so
     // such variables can't be marked as DSO local. (Native TLS variables
     // can't be dllimported at all, though.)
     if (GV->isDeclarationForLinker() && isa<llvm::GlobalVariable>(GV) &&
         (!GV->isThreadLocal() || CGM.getCodeGenOpts().EmulatedTLS))
       return false;
   }
 
   // On COFF, don't mark 'extern_weak' symbols as DSO local. If these symbols
   // remain unresolved in the link, they can be resolved to zero, which is
   // outside the current DSO.
   if (TT.isOSBinFormatCOFF() && GV->hasExternalWeakLinkage())
     return false;
 
   // Every other GV is local on COFF.
   // Make an exception for windows OS in the triple: Some firmware builds use
   // *-win32-macho triples. This (accidentally?) produced windows relocations
   // without GOT tables in older clang versions; Keep this behaviour.
   // FIXME: even thread local variables?
   if (TT.isOSBinFormatCOFF() || (TT.isOSWindows() && TT.isOSBinFormatMachO()))
     return true;
 
   // Only handle COFF and ELF for now.
   if (!TT.isOSBinFormatELF())
     return false;
 
   // If this is not an executable, don't assume anything is local.
   const auto &CGOpts = CGM.getCodeGenOpts();
   llvm::Reloc::Model RM = CGOpts.RelocationModel;
   const auto &LOpts = CGM.getLangOpts();
   if (RM != llvm::Reloc::Static && !LOpts.PIE) {
     // On ELF, if -fno-semantic-interposition is specified and the target
     // supports local aliases, there will be neither CC1
     // -fsemantic-interposition nor -fhalf-no-semantic-interposition. Set
     // dso_local on the function if using a local alias is preferable (can avoid
     // PLT indirection).
     if (!(isa<llvm::Function>(GV) && GV->canBenefitFromLocalAlias()))
       return false;
     return !(CGM.getLangOpts().SemanticInterposition ||
              CGM.getLangOpts().HalfNoSemanticInterposition);
   }
 
   // A definition cannot be preempted from an executable.
   if (!GV->isDeclarationForLinker())
     return true;
 
   // Most PIC code sequences that assume that a symbol is local cannot produce a
   // 0 if it turns out the symbol is undefined. While this is ABI and relocation
   // depended, it seems worth it to handle it here.
   if (RM == llvm::Reloc::PIC_ && GV->hasExternalWeakLinkage())
     return false;
 
   // PowerPC64 prefers TOC indirection to avoid copy relocations.
   if (TT.isPPC64())
     return false;
 
   if (CGOpts.DirectAccessExternalData) {
     // If -fdirect-access-external-data (default for -fno-pic), set dso_local
     // for non-thread-local variables. If the symbol is not defined in the
     // executable, a copy relocation will be needed at link time. dso_local is
     // excluded for thread-local variables because they generally don't support
     // copy relocations.
     if (auto *Var = dyn_cast<llvm::GlobalVariable>(GV))
       if (!Var->isThreadLocal())
         return true;
 
     // -fno-pic sets dso_local on a function declaration to allow direct
     // accesses when taking its address (similar to a data symbol). If the
     // function is not defined in the executable, a canonical PLT entry will be
     // needed at link time. -fno-direct-access-external-data can avoid the
     // canonical PLT entry. We don't generalize this condition to -fpie/-fpic as
     // it could just cause trouble without providing perceptible benefits.
     if (isa<llvm::Function>(GV) && !CGOpts.NoPLT && RM == llvm::Reloc::Static)
       return true;
   }
 
   // If we can use copy relocations we can assume it is local.
 
   // Otherwise don't assume it is local.
   return false;
 }
 
 void CodeGenModule::setDSOLocal(llvm::GlobalValue *GV) const {
   GV->setDSOLocal(shouldAssumeDSOLocal(*this, GV));
 }
 
 void CodeGenModule::setDLLImportDLLExport(llvm::GlobalValue *GV,
                                           GlobalDecl GD) const {
   const auto *D = dyn_cast<NamedDecl>(GD.getDecl());
   // C++ destructors have a few C++ ABI specific special cases.
   if (const auto *Dtor = dyn_cast_or_null<CXXDestructorDecl>(D)) {
     getCXXABI().setCXXDestructorDLLStorage(GV, Dtor, GD.getDtorType());
     return;
   }
   setDLLImportDLLExport(GV, D);
 }
 
 void CodeGenModule::setDLLImportDLLExport(llvm::GlobalValue *GV,
                                           const NamedDecl *D) const {
   if (D && D->isExternallyVisible()) {
     if (D->hasAttr<DLLImportAttr>())
       GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass);
     else if (D->hasAttr<DLLExportAttr>() && !GV->isDeclarationForLinker())
       GV->setDLLStorageClass(llvm::GlobalVariable::DLLExportStorageClass);
   }
 }
 
 void CodeGenModule::setGVProperties(llvm::GlobalValue *GV,
                                     GlobalDecl GD) const {
   setDLLImportDLLExport(GV, GD);
   setGVPropertiesAux(GV, dyn_cast<NamedDecl>(GD.getDecl()));
 }
 
 void CodeGenModule::setGVProperties(llvm::GlobalValue *GV,
                                     const NamedDecl *D) const {
   setDLLImportDLLExport(GV, D);
   setGVPropertiesAux(GV, D);
 }
 
 void CodeGenModule::setGVPropertiesAux(llvm::GlobalValue *GV,
                                        const NamedDecl *D) const {
   setGlobalVisibility(GV, D);
   setDSOLocal(GV);
   GV->setPartition(CodeGenOpts.SymbolPartition);
 }
 
 static llvm::GlobalVariable::ThreadLocalMode GetLLVMTLSModel(StringRef S) {
   return llvm::StringSwitch<llvm::GlobalVariable::ThreadLocalMode>(S)
       .Case("global-dynamic", llvm::GlobalVariable::GeneralDynamicTLSModel)
       .Case("local-dynamic", llvm::GlobalVariable::LocalDynamicTLSModel)
       .Case("initial-exec", llvm::GlobalVariable::InitialExecTLSModel)
       .Case("local-exec", llvm::GlobalVariable::LocalExecTLSModel);
 }
 
 llvm::GlobalVariable::ThreadLocalMode
 CodeGenModule::GetDefaultLLVMTLSModel() const {
   switch (CodeGenOpts.getDefaultTLSModel()) {
   case CodeGenOptions::GeneralDynamicTLSModel:
     return llvm::GlobalVariable::GeneralDynamicTLSModel;
   case CodeGenOptions::LocalDynamicTLSModel:
     return llvm::GlobalVariable::LocalDynamicTLSModel;
   case CodeGenOptions::InitialExecTLSModel:
     return llvm::GlobalVariable::InitialExecTLSModel;
   case CodeGenOptions::LocalExecTLSModel:
     return llvm::GlobalVariable::LocalExecTLSModel;
   }
   llvm_unreachable("Invalid TLS model!");
 }
 
 void CodeGenModule::setTLSMode(llvm::GlobalValue *GV, const VarDecl &D) const {
   assert(D.getTLSKind() && "setting TLS mode on non-TLS var!");
 
   llvm::GlobalValue::ThreadLocalMode TLM;
   TLM = GetDefaultLLVMTLSModel();
 
   // Override the TLS model if it is explicitly specified.
   if (const TLSModelAttr *Attr = D.getAttr<TLSModelAttr>()) {
     TLM = GetLLVMTLSModel(Attr->getModel());
   }
 
   GV->setThreadLocalMode(TLM);
 }
 
 static std::string getCPUSpecificMangling(const CodeGenModule &CGM,
                                           StringRef Name) {
   const TargetInfo &Target = CGM.getTarget();
   return (Twine('.') + Twine(Target.CPUSpecificManglingCharacter(Name))).str();
 }
 
 static void AppendCPUSpecificCPUDispatchMangling(const CodeGenModule &CGM,
                                                  const CPUSpecificAttr *Attr,
                                                  unsigned CPUIndex,
                                                  raw_ostream &Out) {
   // cpu_specific gets the current name, dispatch gets the resolver if IFunc is
   // supported.
   if (Attr)
     Out << getCPUSpecificMangling(CGM, Attr->getCPUName(CPUIndex)->getName());
   else if (CGM.getTarget().supportsIFunc())
     Out << ".resolver";
 }
 
 static void AppendTargetMangling(const CodeGenModule &CGM,
                                  const TargetAttr *Attr, raw_ostream &Out) {
   if (Attr->isDefaultVersion())
     return;
 
   Out << '.';
   const TargetInfo &Target = CGM.getTarget();
   ParsedTargetAttr Info =
       Attr->parse([&Target](StringRef LHS, StringRef RHS) {
         // Multiversioning doesn't allow "no-${feature}", so we can
         // only have "+" prefixes here.
         assert(LHS.startswith("+") && RHS.startswith("+") &&
                "Features should always have a prefix.");
         return Target.multiVersionSortPriority(LHS.substr(1)) >
                Target.multiVersionSortPriority(RHS.substr(1));
       });
 
   bool IsFirst = true;
 
   if (!Info.Architecture.empty()) {
     IsFirst = false;
     Out << "arch_" << Info.Architecture;
   }
 
   for (StringRef Feat : Info.Features) {
     if (!IsFirst)
       Out << '_';
     IsFirst = false;
     Out << Feat.substr(1);
   }
 }
 
 // Returns true if GD is a function decl with internal linkage and
 // needs a unique suffix after the mangled name.
 static bool isUniqueInternalLinkageDecl(GlobalDecl GD,
                                         CodeGenModule &CGM) {
   const Decl *D = GD.getDecl();
   return !CGM.getModuleNameHash().empty() && isa<FunctionDecl>(D) &&
          (CGM.getFunctionLinkage(GD) == llvm::GlobalValue::InternalLinkage);
 }
 
 static void AppendTargetClonesMangling(const CodeGenModule &CGM,
                                        const TargetClonesAttr *Attr,
                                        unsigned VersionIndex,
                                        raw_ostream &Out) {
   Out << '.';
   StringRef FeatureStr = Attr->getFeatureStr(VersionIndex);
   if (FeatureStr.startswith("arch="))
     Out << "arch_" << FeatureStr.substr(sizeof("arch=") - 1);
   else
     Out << FeatureStr;
 
   Out << '.' << Attr->getMangledIndex(VersionIndex);
 }
 
 static std::string getMangledNameImpl(CodeGenModule &CGM, GlobalDecl GD,
                                       const NamedDecl *ND,
                                       bool OmitMultiVersionMangling = false) {
   SmallString<256> Buffer;
   llvm::raw_svector_ostream Out(Buffer);
   MangleContext &MC = CGM.getCXXABI().getMangleContext();
   if (!CGM.getModuleNameHash().empty())
     MC.needsUniqueInternalLinkageNames();
   bool ShouldMangle = MC.shouldMangleDeclName(ND);
   if (ShouldMangle)
     MC.mangleName(GD.getWithDecl(ND), Out);
   else {
     IdentifierInfo *II = ND->getIdentifier();
     assert(II && "Attempt to mangle unnamed decl.");
     const auto *FD = dyn_cast<FunctionDecl>(ND);
 
     if (FD &&
         FD->getType()->castAs<FunctionType>()->getCallConv() == CC_X86RegCall) {
       Out << "__regcall3__" << II->getName();
     } else if (FD && FD->hasAttr<CUDAGlobalAttr>() &&
                GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
       Out << "__device_stub__" << II->getName();
     } else {
       Out << II->getName();
     }
   }
 
   // Check if the module name hash should be appended for internal linkage
   // symbols.   This should come before multi-version target suffixes are
   // appended. This is to keep the name and module hash suffix of the
   // internal linkage function together.  The unique suffix should only be
   // added when name mangling is done to make sure that the final name can
   // be properly demangled.  For example, for C functions without prototypes,
   // name mangling is not done and the unique suffix should not be appeneded
   // then.
   if (ShouldMangle && isUniqueInternalLinkageDecl(GD, CGM)) {
     assert(CGM.getCodeGenOpts().UniqueInternalLinkageNames &&
            "Hash computed when not explicitly requested");
     Out << CGM.getModuleNameHash();
   }
 
   if (const auto *FD = dyn_cast<FunctionDecl>(ND))
     if (FD->isMultiVersion() && !OmitMultiVersionMangling) {
       switch (FD->getMultiVersionKind()) {
       case MultiVersionKind::CPUDispatch:
       case MultiVersionKind::CPUSpecific:
         AppendCPUSpecificCPUDispatchMangling(CGM,
                                              FD->getAttr<CPUSpecificAttr>(),
                                              GD.getMultiVersionIndex(), Out);
         break;
       case MultiVersionKind::Target:
         AppendTargetMangling(CGM, FD->getAttr<TargetAttr>(), Out);
         break;
       case MultiVersionKind::TargetClones:
         AppendTargetClonesMangling(CGM, FD->getAttr<TargetClonesAttr>(),
                                    GD.getMultiVersionIndex(), Out);
         break;
       case MultiVersionKind::None:
         llvm_unreachable("None multiversion type isn't valid here");
       }
     }
 
   // Make unique name for device side static file-scope variable for HIP.
   if (CGM.getContext().shouldExternalizeStaticVar(ND) &&
       CGM.getLangOpts().GPURelocatableDeviceCode &&
       CGM.getLangOpts().CUDAIsDevice && !CGM.getLangOpts().CUID.empty())
-    CGM.printPostfixForExternalizedStaticVar(Out);
+    CGM.printPostfixForExternalizedDecl(Out, ND);
   return std::string(Out.str());
 }
 
 void CodeGenModule::UpdateMultiVersionNames(GlobalDecl GD,
                                             const FunctionDecl *FD,
                                             StringRef &CurName) {
   if (!FD->isMultiVersion())
     return;
 
   // Get the name of what this would be without the 'target' attribute.  This
   // allows us to lookup the version that was emitted when this wasn't a
   // multiversion function.
   std::string NonTargetName =
       getMangledNameImpl(*this, GD, FD, /*OmitMultiVersionMangling=*/true);
   GlobalDecl OtherGD;
   if (lookupRepresentativeDecl(NonTargetName, OtherGD)) {
     assert(OtherGD.getCanonicalDecl()
                .getDecl()
                ->getAsFunction()
                ->isMultiVersion() &&
            "Other GD should now be a multiversioned function");
     // OtherFD is the version of this function that was mangled BEFORE
     // becoming a MultiVersion function.  It potentially needs to be updated.
     const FunctionDecl *OtherFD = OtherGD.getCanonicalDecl()
                                       .getDecl()
                                       ->getAsFunction()
                                       ->getMostRecentDecl();
     std::string OtherName = getMangledNameImpl(*this, OtherGD, OtherFD);
     // This is so that if the initial version was already the 'default'
     // version, we don't try to update it.
     if (OtherName != NonTargetName) {
       // Remove instead of erase, since others may have stored the StringRef
       // to this.
       const auto ExistingRecord = Manglings.find(NonTargetName);
       if (ExistingRecord != std::end(Manglings))
         Manglings.remove(&(*ExistingRecord));
       auto Result = Manglings.insert(std::make_pair(OtherName, OtherGD));
       StringRef OtherNameRef = MangledDeclNames[OtherGD.getCanonicalDecl()] =
           Result.first->first();
       // If this is the current decl is being created, make sure we update the name.
       if (GD.getCanonicalDecl() == OtherGD.getCanonicalDecl())
         CurName = OtherNameRef;
       if (llvm::GlobalValue *Entry = GetGlobalValue(NonTargetName))
         Entry->setName(OtherName);
     }
   }
 }
 
 StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
   GlobalDecl CanonicalGD = GD.getCanonicalDecl();
 
   // Some ABIs don't have constructor variants.  Make sure that base and
   // complete constructors get mangled the same.
   if (const auto *CD = dyn_cast<CXXConstructorDecl>(CanonicalGD.getDecl())) {
     if (!getTarget().getCXXABI().hasConstructorVariants()) {
       CXXCtorType OrigCtorType = GD.getCtorType();
       assert(OrigCtorType == Ctor_Base || OrigCtorType == Ctor_Complete);
       if (OrigCtorType == Ctor_Base)
         CanonicalGD = GlobalDecl(CD, Ctor_Complete);
     }
   }
 
   // In CUDA/HIP device compilation with -fgpu-rdc, the mangled name of a
   // static device variable depends on whether the variable is referenced by
   // a host or device host function. Therefore the mangled name cannot be
   // cached.
   if (!LangOpts.CUDAIsDevice ||
       !getContext().mayExternalizeStaticVar(GD.getDecl())) {
     auto FoundName = MangledDeclNames.find(CanonicalGD);
     if (FoundName != MangledDeclNames.end())
       return FoundName->second;
   }
 
   // Keep the first result in the case of a mangling collision.
   const auto *ND = cast<NamedDecl>(GD.getDecl());
   std::string MangledName = getMangledNameImpl(*this, GD, ND);
 
   // Ensure either we have different ABIs between host and device compilations,
   // says host compilation following MSVC ABI but device compilation follows
   // Itanium C++ ABI or, if they follow the same ABI, kernel names after
   // mangling should be the same after name stubbing. The later checking is
   // very important as the device kernel name being mangled in host-compilation
   // is used to resolve the device binaries to be executed. Inconsistent naming
   // result in undefined behavior. Even though we cannot check that naming
   // directly between host- and device-compilations, the host- and
   // device-mangling in host compilation could help catching certain ones.
   assert(!isa<FunctionDecl>(ND) || !ND->hasAttr<CUDAGlobalAttr>() ||
-         getLangOpts().CUDAIsDevice ||
+         getContext().shouldExternalizeStaticVar(ND) || getLangOpts().CUDAIsDevice ||
          (getContext().getAuxTargetInfo() &&
           (getContext().getAuxTargetInfo()->getCXXABI() !=
            getContext().getTargetInfo().getCXXABI())) ||
          getCUDARuntime().getDeviceSideName(ND) ==
              getMangledNameImpl(
                  *this,
                  GD.getWithKernelReferenceKind(KernelReferenceKind::Kernel),
                  ND));
 
   auto Result = Manglings.insert(std::make_pair(MangledName, GD));
   return MangledDeclNames[CanonicalGD] = Result.first->first();
 }
 
 StringRef CodeGenModule::getBlockMangledName(GlobalDecl GD,
                                              const BlockDecl *BD) {
   MangleContext &MangleCtx = getCXXABI().getMangleContext();
   const Decl *D = GD.getDecl();
 
   SmallString<256> Buffer;
   llvm::raw_svector_ostream Out(Buffer);
   if (!D)
     MangleCtx.mangleGlobalBlock(BD,
       dyn_cast_or_null<VarDecl>(initializedGlobalDecl.getDecl()), Out);
   else if (const auto *CD = dyn_cast<CXXConstructorDecl>(D))
     MangleCtx.mangleCtorBlock(CD, GD.getCtorType(), BD, Out);
   else if (const auto *DD = dyn_cast<CXXDestructorDecl>(D))
     MangleCtx.mangleDtorBlock(DD, GD.getDtorType(), BD, Out);
   else
     MangleCtx.mangleBlock(cast<DeclContext>(D), BD, Out);
 
   auto Result = Manglings.insert(std::make_pair(Out.str(), BD));
   return Result.first->first();
 }
 
 llvm::GlobalValue *CodeGenModule::GetGlobalValue(StringRef Name) {
   return getModule().getNamedValue(Name);
 }
 
 /// AddGlobalCtor - Add a function to the list that will be called before
 /// main() runs.
 void CodeGenModule::AddGlobalCtor(llvm::Function *Ctor, int Priority,
                                   llvm::Constant *AssociatedData) {
   // FIXME: Type coercion of void()* types.
   GlobalCtors.push_back(Structor(Priority, Ctor, AssociatedData));
 }
 
 /// AddGlobalDtor - Add a function to the list that will be called
 /// when the module is unloaded.
 void CodeGenModule::AddGlobalDtor(llvm::Function *Dtor, int Priority,
                                   bool IsDtorAttrFunc) {
   if (CodeGenOpts.RegisterGlobalDtorsWithAtExit &&
       (!getContext().getTargetInfo().getTriple().isOSAIX() || IsDtorAttrFunc)) {
     DtorsUsingAtExit[Priority].push_back(Dtor);
     return;
   }
 
   // FIXME: Type coercion of void()* types.
   GlobalDtors.push_back(Structor(Priority, Dtor, nullptr));
 }
 
 void CodeGenModule::EmitCtorList(CtorList &Fns, const char *GlobalName) {
   if (Fns.empty()) return;
 
   // Ctor function type is void()*.
   llvm::FunctionType* CtorFTy = llvm::FunctionType::get(VoidTy, false);
   llvm::Type *CtorPFTy = llvm::PointerType::get(CtorFTy,
       TheModule.getDataLayout().getProgramAddressSpace());
 
   // Get the type of a ctor entry, { i32, void ()*, i8* }.
   llvm::StructType *CtorStructTy = llvm::StructType::get(
       Int32Ty, CtorPFTy, VoidPtrTy);
 
   // Construct the constructor and destructor arrays.
   ConstantInitBuilder builder(*this);
   auto ctors = builder.beginArray(CtorStructTy);
   for (const auto &I : Fns) {
     auto ctor = ctors.beginStruct(CtorStructTy);
     ctor.addInt(Int32Ty, I.Priority);
     ctor.add(llvm::ConstantExpr::getBitCast(I.Initializer, CtorPFTy));
     if (I.AssociatedData)
       ctor.add(llvm::ConstantExpr::getBitCast(I.AssociatedData, VoidPtrTy));
     else
       ctor.addNullPointer(VoidPtrTy);
     ctor.finishAndAddTo(ctors);
   }
 
   auto list =
     ctors.finishAndCreateGlobal(GlobalName, getPointerAlign(),
                                 /*constant*/ false,
                                 llvm::GlobalValue::AppendingLinkage);
 
   // The LTO linker doesn't seem to like it when we set an alignment
   // on appending variables.  Take it off as a workaround.
   list->setAlignment(llvm::None);
 
   Fns.clear();
 }
 
 llvm::GlobalValue::LinkageTypes
 CodeGenModule::getFunctionLinkage(GlobalDecl GD) {
   const auto *D = cast<FunctionDecl>(GD.getDecl());
 
   GVALinkage Linkage = getContext().GetGVALinkageForFunction(D);
 
   if (const auto *Dtor = dyn_cast<CXXDestructorDecl>(D))
     return getCXXABI().getCXXDestructorLinkage(Linkage, Dtor, GD.getDtorType());
 
   if (isa<CXXConstructorDecl>(D) &&
       cast<CXXConstructorDecl>(D)->isInheritingConstructor() &&
       Context.getTargetInfo().getCXXABI().isMicrosoft()) {
     // Our approach to inheriting constructors is fundamentally different from
     // that used by the MS ABI, so keep our inheriting constructor thunks
     // internal rather than trying to pick an unambiguous mangling for them.
     return llvm::GlobalValue::InternalLinkage;
   }
 
   return getLLVMLinkageForDeclarator(D, Linkage, /*IsConstantVariable=*/false);
 }
 
 llvm::ConstantInt *CodeGenModule::CreateCrossDsoCfiTypeId(llvm::Metadata *MD) {
   llvm::MDString *MDS = dyn_cast<llvm::MDString>(MD);
   if (!MDS) return nullptr;
 
   return llvm::ConstantInt::get(Int64Ty, llvm::MD5Hash(MDS->getString()));
 }
 
 void CodeGenModule::SetLLVMFunctionAttributes(GlobalDecl GD,
                                               const CGFunctionInfo &Info,
                                               llvm::Function *F, bool IsThunk) {
   unsigned CallingConv;
   llvm::AttributeList PAL;
   ConstructAttributeList(F->getName(), Info, GD, PAL, CallingConv,
                          /*AttrOnCallSite=*/false, IsThunk);
   F->setAttributes(PAL);
   F->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
 }
 
 static void removeImageAccessQualifier(std::string& TyName) {
   std::string ReadOnlyQual("__read_only");
   std::string::size_type ReadOnlyPos = TyName.find(ReadOnlyQual);
   if (ReadOnlyPos != std::string::npos)
     // "+ 1" for the space after access qualifier.
     TyName.erase(ReadOnlyPos, ReadOnlyQual.size() + 1);
   else {
     std::string WriteOnlyQual("__write_only");
     std::string::size_type WriteOnlyPos = TyName.find(WriteOnlyQual);
     if (WriteOnlyPos != std::string::npos)
       TyName.erase(WriteOnlyPos, WriteOnlyQual.size() + 1);
     else {
       std::string ReadWriteQual("__read_write");
       std::string::size_type ReadWritePos = TyName.find(ReadWriteQual);
       if (ReadWritePos != std::string::npos)
         TyName.erase(ReadWritePos, ReadWriteQual.size() + 1);
     }
   }
 }
 
 // Returns the address space id that should be produced to the
 // kernel_arg_addr_space metadata. This is always fixed to the ids
 // as specified in the SPIR 2.0 specification in order to differentiate
 // for example in clGetKernelArgInfo() implementation between the address
 // spaces with targets without unique mapping to the OpenCL address spaces
 // (basically all single AS CPUs).
 static unsigned ArgInfoAddressSpace(LangAS AS) {
   switch (AS) {
   case LangAS::opencl_global:
     return 1;
   case LangAS::opencl_constant:
     return 2;
   case LangAS::opencl_local:
     return 3;
   case LangAS::opencl_generic:
     return 4; // Not in SPIR 2.0 specs.
   case LangAS::opencl_global_device:
     return 5;
   case LangAS::opencl_global_host:
     return 6;
   default:
     return 0; // Assume private.
   }
 }
 
 void CodeGenModule::GenOpenCLArgMetadata(llvm::Function *Fn,
                                          const FunctionDecl *FD,
                                          CodeGenFunction *CGF) {
   assert(((FD && CGF) || (!FD && !CGF)) &&
          "Incorrect use - FD and CGF should either be both null or not!");
   // Create MDNodes that represent the kernel arg metadata.
   // Each MDNode is a list in the form of "key", N number of values which is
   // the same number of values as their are kernel arguments.
 
   const PrintingPolicy &Policy = Context.getPrintingPolicy();
 
   // MDNode for the kernel argument address space qualifiers.
   SmallVector<llvm::Metadata *, 8> addressQuals;
 
   // MDNode for the kernel argument access qualifiers (images only).
   SmallVector<llvm::Metadata *, 8> accessQuals;
 
   // MDNode for the kernel argument type names.
   SmallVector<llvm::Metadata *, 8> argTypeNames;
 
   // MDNode for the kernel argument base type names.
   SmallVector<llvm::Metadata *, 8> argBaseTypeNames;
 
   // MDNode for the kernel argument type qualifiers.
   SmallVector<llvm::Metadata *, 8> argTypeQuals;
 
   // MDNode for the kernel argument names.
   SmallVector<llvm::Metadata *, 8> argNames;
 
   if (FD && CGF)
     for (unsigned i = 0, e = FD->getNumParams(); i != e; ++i) {
       const ParmVarDecl *parm = FD->getParamDecl(i);
       QualType ty = parm->getType();
       std::string typeQuals;
 
       // Get image and pipe access qualifier:
       if (ty->isImageType() || ty->isPipeType()) {
         const Decl *PDecl = parm;
         if (auto *TD = dyn_cast<TypedefType>(ty))
           PDecl = TD->getDecl();
         const OpenCLAccessAttr *A = PDecl->getAttr<OpenCLAccessAttr>();
         if (A && A->isWriteOnly())
           accessQuals.push_back(llvm::MDString::get(VMContext, "write_only"));
         else if (A && A->isReadWrite())
           accessQuals.push_back(llvm::MDString::get(VMContext, "read_write"));
         else
           accessQuals.push_back(llvm::MDString::get(VMContext, "read_only"));
       } else
         accessQuals.push_back(llvm::MDString::get(VMContext, "none"));
 
       // Get argument name.
       argNames.push_back(llvm::MDString::get(VMContext, parm->getName()));
 
       auto getTypeSpelling = [&](QualType Ty) {
         auto typeName = Ty.getUnqualifiedType().getAsString(Policy);
 
         if (Ty.isCanonical()) {
           StringRef typeNameRef = typeName;
           // Turn "unsigned type" to "utype"
           if (typeNameRef.consume_front("unsigned "))
             return std::string("u") + typeNameRef.str();
           if (typeNameRef.consume_front("signed "))
             return typeNameRef.str();
         }
 
         return typeName;
       };
 
       if (ty->isPointerType()) {
         QualType pointeeTy = ty->getPointeeType();
 
         // Get address qualifier.
         addressQuals.push_back(
             llvm::ConstantAsMetadata::get(CGF->Builder.getInt32(
                 ArgInfoAddressSpace(pointeeTy.getAddressSpace()))));
 
         // Get argument type name.
         std::string typeName = getTypeSpelling(pointeeTy) + "*";
         std::string baseTypeName =
             getTypeSpelling(pointeeTy.getCanonicalType()) + "*";
         argTypeNames.push_back(llvm::MDString::get(VMContext, typeName));
         argBaseTypeNames.push_back(
             llvm::MDString::get(VMContext, baseTypeName));
 
         // Get argument type qualifiers:
         if (ty.isRestrictQualified())
           typeQuals = "restrict";
         if (pointeeTy.isConstQualified() ||
             (pointeeTy.getAddressSpace() == LangAS::opencl_constant))
           typeQuals += typeQuals.empty() ? "const" : " const";
         if (pointeeTy.isVolatileQualified())
           typeQuals += typeQuals.empty() ? "volatile" : " volatile";
       } else {
         uint32_t AddrSpc = 0;
         bool isPipe = ty->isPipeType();
         if (ty->isImageType() || isPipe)
           AddrSpc = ArgInfoAddressSpace(LangAS::opencl_global);
 
         addressQuals.push_back(
             llvm::ConstantAsMetadata::get(CGF->Builder.getInt32(AddrSpc)));
 
         // Get argument type name.
         ty = isPipe ? ty->castAs<PipeType>()->getElementType() : ty;
         std::string typeName = getTypeSpelling(ty);
         std::string baseTypeName = getTypeSpelling(ty.getCanonicalType());
 
         // Remove access qualifiers on images
         // (as they are inseparable from type in clang implementation,
         // but OpenCL spec provides a special query to get access qualifier
         // via clGetKernelArgInfo with CL_KERNEL_ARG_ACCESS_QUALIFIER):
         if (ty->isImageType()) {
           removeImageAccessQualifier(typeName);
           removeImageAccessQualifier(baseTypeName);
         }
 
         argTypeNames.push_back(llvm::MDString::get(VMContext, typeName));
         argBaseTypeNames.push_back(
             llvm::MDString::get(VMContext, baseTypeName));
 
         if (isPipe)
           typeQuals = "pipe";
       }
       argTypeQuals.push_back(llvm::MDString::get(VMContext, typeQuals));
     }
 
   Fn->setMetadata("kernel_arg_addr_space",
                   llvm::MDNode::get(VMContext, addressQuals));
   Fn->setMetadata("kernel_arg_access_qual",
                   llvm::MDNode::get(VMContext, accessQuals));
   Fn->setMetadata("kernel_arg_type",
                   llvm::MDNode::get(VMContext, argTypeNames));
   Fn->setMetadata("kernel_arg_base_type",
                   llvm::MDNode::get(VMContext, argBaseTypeNames));
   Fn->setMetadata("kernel_arg_type_qual",
                   llvm::MDNode::get(VMContext, argTypeQuals));
   if (getCodeGenOpts().EmitOpenCLArgMetadata)
     Fn->setMetadata("kernel_arg_name",
                     llvm::MDNode::get(VMContext, argNames));
 }
 
 /// Determines whether the language options require us to model
 /// unwind exceptions.  We treat -fexceptions as mandating this
 /// except under the fragile ObjC ABI with only ObjC exceptions
 /// enabled.  This means, for example, that C with -fexceptions
 /// enables this.
 static bool hasUnwindExceptions(const LangOptions &LangOpts) {
   // If exceptions are completely disabled, obviously this is false.
   if (!LangOpts.Exceptions) return false;
 
   // If C++ exceptions are enabled, this is true.
   if (LangOpts.CXXExceptions) return true;
 
   // If ObjC exceptions are enabled, this depends on the ABI.
   if (LangOpts.ObjCExceptions) {
     return LangOpts.ObjCRuntime.hasUnwindExceptions();
   }
 
   return true;
 }
 
 static bool requiresMemberFunctionPointerTypeMetadata(CodeGenModule &CGM,
                                                       const CXXMethodDecl *MD) {
   // Check that the type metadata can ever actually be used by a call.
   if (!CGM.getCodeGenOpts().LTOUnit ||
       !CGM.HasHiddenLTOVisibility(MD->getParent()))
     return false;
 
   // Only functions whose address can be taken with a member function pointer
   // need this sort of type metadata.
   return !MD->isStatic() && !MD->isVirtual() && !isa<CXXConstructorDecl>(MD) &&
          !isa<CXXDestructorDecl>(MD);
 }
 
 std::vector<const CXXRecordDecl *>
 CodeGenModule::getMostBaseClasses(const CXXRecordDecl *RD) {
   llvm::SetVector<const CXXRecordDecl *> MostBases;
 
   std::function<void (const CXXRecordDecl *)> CollectMostBases;
   CollectMostBases = [&](const CXXRecordDecl *RD) {
     if (RD->getNumBases() == 0)
       MostBases.insert(RD);
     for (const CXXBaseSpecifier &B : RD->bases())
       CollectMostBases(B.getType()->getAsCXXRecordDecl());
   };
   CollectMostBases(RD);
   return MostBases.takeVector();
 }
 
 void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
                                                            llvm::Function *F) {
   llvm::AttrBuilder B(F->getContext());
 
   if (CodeGenOpts.UnwindTables)
     B.addAttribute(llvm::Attribute::UWTable);
 
   if (CodeGenOpts.StackClashProtector)
     B.addAttribute("probe-stack", "inline-asm");
 
   if (!hasUnwindExceptions(LangOpts))
     B.addAttribute(llvm::Attribute::NoUnwind);
 
   if (!D || !D->hasAttr<NoStackProtectorAttr>()) {
     if (LangOpts.getStackProtector() == LangOptions::SSPOn)
       B.addAttribute(llvm::Attribute::StackProtect);
     else if (LangOpts.getStackProtector() == LangOptions::SSPStrong)
       B.addAttribute(llvm::Attribute::StackProtectStrong);
     else if (LangOpts.getStackProtector() == LangOptions::SSPReq)
       B.addAttribute(llvm::Attribute::StackProtectReq);
   }
 
   if (!D) {
     // If we don't have a declaration to control inlining, the function isn't
     // explicitly marked as alwaysinline for semantic reasons, and inlining is
     // disabled, mark the function as noinline.
     if (!F->hasFnAttribute(llvm::Attribute::AlwaysInline) &&
         CodeGenOpts.getInlining() == CodeGenOptions::OnlyAlwaysInlining)
       B.addAttribute(llvm::Attribute::NoInline);
 
     F->addFnAttrs(B);
     return;
   }
 
   // Track whether we need to add the optnone LLVM attribute,
   // starting with the default for this optimization level.
   bool ShouldAddOptNone =
       !CodeGenOpts.DisableO0ImplyOptNone && CodeGenOpts.OptimizationLevel == 0;
   // We can't add optnone in the following cases, it won't pass the verifier.
   ShouldAddOptNone &= !D->hasAttr<MinSizeAttr>();
   ShouldAddOptNone &= !D->hasAttr<AlwaysInlineAttr>();
 
   // Add optnone, but do so only if the function isn't always_inline.
   if ((ShouldAddOptNone || D->hasAttr<OptimizeNoneAttr>()) &&
       !F->hasFnAttribute(llvm::Attribute::AlwaysInline)) {
     B.addAttribute(llvm::Attribute::OptimizeNone);
 
     // OptimizeNone implies noinline; we should not be inlining such functions.
     B.addAttribute(llvm::Attribute::NoInline);
 
     // We still need to handle naked functions even though optnone subsumes
     // much of their semantics.
     if (D->hasAttr<NakedAttr>())
       B.addAttribute(llvm::Attribute::Naked);
 
     // OptimizeNone wins over OptimizeForSize and MinSize.
     F->removeFnAttr(llvm::Attribute::OptimizeForSize);
     F->removeFnAttr(llvm::Attribute::MinSize);
   } else if (D->hasAttr<NakedAttr>()) {
     // Naked implies noinline: we should not be inlining such functions.
     B.addAttribute(llvm::Attribute::Naked);
     B.addAttribute(llvm::Attribute::NoInline);
   } else if (D->hasAttr<NoDuplicateAttr>()) {
     B.addAttribute(llvm::Attribute::NoDuplicate);
   } else if (D->hasAttr<NoInlineAttr>() && !F->hasFnAttribute(llvm::Attribute::AlwaysInline)) {
     // Add noinline if the function isn't always_inline.
     B.addAttribute(llvm::Attribute::NoInline);
   } else if (D->hasAttr<AlwaysInlineAttr>() &&
              !F->hasFnAttribute(llvm::Attribute::NoInline)) {
     // (noinline wins over always_inline, and we can't specify both in IR)
     B.addAttribute(llvm::Attribute::AlwaysInline);
   } else if (CodeGenOpts.getInlining() == CodeGenOptions::OnlyAlwaysInlining) {
     // If we're not inlining, then force everything that isn't always_inline to
     // carry an explicit noinline attribute.
     if (!F->hasFnAttribute(llvm::Attribute::AlwaysInline))
       B.addAttribute(llvm::Attribute::NoInline);
   } else {
     // Otherwise, propagate the inline hint attribute and potentially use its
     // absence to mark things as noinline.
     if (auto *FD = dyn_cast<FunctionDecl>(D)) {
       // Search function and template pattern redeclarations for inline.
       auto CheckForInline = [](const FunctionDecl *FD) {
         auto CheckRedeclForInline = [](const FunctionDecl *Redecl) {
           return Redecl->isInlineSpecified();
         };
         if (any_of(FD->redecls(), CheckRedeclForInline))
           return true;
         const FunctionDecl *Pattern = FD->getTemplateInstantiationPattern();
         if (!Pattern)
           return false;
         return any_of(Pattern->redecls(), CheckRedeclForInline);
       };
       if (CheckForInline(FD)) {
         B.addAttribute(llvm::Attribute::InlineHint);
       } else if (CodeGenOpts.getInlining() ==
                      CodeGenOptions::OnlyHintInlining &&
                  !FD->isInlined() &&
                  !F->hasFnAttribute(llvm::Attribute::AlwaysInline)) {
         B.addAttribute(llvm::Attribute::NoInline);
       }
     }
   }
 
   // Add other optimization related attributes if we are optimizing this
   // function.
   if (!D->hasAttr<OptimizeNoneAttr>()) {
     if (D->hasAttr<ColdAttr>()) {
       if (!ShouldAddOptNone)
         B.addAttribute(llvm::Attribute::OptimizeForSize);
       B.addAttribute(llvm::Attribute::Cold);
     }
     if (D->hasAttr<HotAttr>())
       B.addAttribute(llvm::Attribute::Hot);
     if (D->hasAttr<MinSizeAttr>())
       B.addAttribute(llvm::Attribute::MinSize);
   }
 
   F->addFnAttrs(B);
 
   unsigned alignment = D->getMaxAlignment() / Context.getCharWidth();
   if (alignment)
     F->setAlignment(llvm::Align(alignment));
 
   if (!D->hasAttr<AlignedAttr>())
     if (LangOpts.FunctionAlignment)
       F->setAlignment(llvm::Align(1ull << LangOpts.FunctionAlignment));
 
   // Some C++ ABIs require 2-byte alignment for member functions, in order to
   // reserve a bit for differentiating between virtual and non-virtual member
   // functions. If the current target's C++ ABI requires this and this is a
   // member function, set its alignment accordingly.
   if (getTarget().getCXXABI().areMemberFunctionsAligned()) {
     if (F->getAlignment() < 2 && isa<CXXMethodDecl>(D))
       F->setAlignment(llvm::Align(2));
   }
 
   // In the cross-dso CFI mode with canonical jump tables, we want !type
   // attributes on definitions only.
   if (CodeGenOpts.SanitizeCfiCrossDso &&
       CodeGenOpts.SanitizeCfiCanonicalJumpTables) {
     if (auto *FD = dyn_cast<FunctionDecl>(D)) {
       // Skip available_externally functions. They won't be codegen'ed in the
       // current module anyway.
       if (getContext().GetGVALinkageForFunction(FD) != GVA_AvailableExternally)
         CreateFunctionTypeMetadataForIcall(FD, F);
     }
   }
 
   // Emit type metadata on member functions for member function pointer checks.
   // These are only ever necessary on definitions; we're guaranteed that the
   // definition will be present in the LTO unit as a result of LTO visibility.
   auto *MD = dyn_cast<CXXMethodDecl>(D);
   if (MD && requiresMemberFunctionPointerTypeMetadata(*this, MD)) {
     for (const CXXRecordDecl *Base : getMostBaseClasses(MD->getParent())) {
       llvm::Metadata *Id =
           CreateMetadataIdentifierForType(Context.getMemberPointerType(
               MD->getType(), Context.getRecordType(Base).getTypePtr()));
       F->addTypeMetadata(0, Id);
     }
   }
 }
 
 void CodeGenModule::setLLVMFunctionFEnvAttributes(const FunctionDecl *D,
                                                   llvm::Function *F) {
   if (D->hasAttr<StrictFPAttr>()) {
     llvm::AttrBuilder FuncAttrs(F->getContext());
     FuncAttrs.addAttribute("strictfp");
     F->addFnAttrs(FuncAttrs);
   }
 }
 
 void CodeGenModule::SetCommonAttributes(GlobalDecl GD, llvm::GlobalValue *GV) {
   const Decl *D = GD.getDecl();
   if (isa_and_nonnull<NamedDecl>(D))
     setGVProperties(GV, GD);
   else
     GV->setVisibility(llvm::GlobalValue::DefaultVisibility);
 
   if (D && D->hasAttr<UsedAttr>())
     addUsedOrCompilerUsedGlobal(GV);
 
   if (CodeGenOpts.KeepStaticConsts && D && isa<VarDecl>(D)) {
     const auto *VD = cast<VarDecl>(D);
     if (VD->getType().isConstQualified() &&
         VD->getStorageDuration() == SD_Static)
       addUsedOrCompilerUsedGlobal(GV);
   }
 }
 
 bool CodeGenModule::GetCPUAndFeaturesAttributes(GlobalDecl GD,
                                                 llvm::AttrBuilder &Attrs) {
   // Add target-cpu and target-features attributes to functions. If
   // we have a decl for the function and it has a target attribute then
   // parse that and add it to the feature set.
   StringRef TargetCPU = getTarget().getTargetOpts().CPU;
   StringRef TuneCPU = getTarget().getTargetOpts().TuneCPU;
   std::vector<std::string> Features;
   const auto *FD = dyn_cast_or_null<FunctionDecl>(GD.getDecl());
   FD = FD ? FD->getMostRecentDecl() : FD;
   const auto *TD = FD ? FD->getAttr<TargetAttr>() : nullptr;
   const auto *SD = FD ? FD->getAttr<CPUSpecificAttr>() : nullptr;
   const auto *TC = FD ? FD->getAttr<TargetClonesAttr>() : nullptr;
   bool AddedAttr = false;
   if (TD || SD || TC) {
     llvm::StringMap<bool> FeatureMap;
     getContext().getFunctionFeatureMap(FeatureMap, GD);
 
     // Produce the canonical string for this set of features.
     for (const llvm::StringMap<bool>::value_type &Entry : FeatureMap)
       Features.push_back((Entry.getValue() ? "+" : "-") + Entry.getKey().str());
 
     // Now add the target-cpu and target-features to the function.
     // While we populated the feature map above, we still need to
     // get and parse the target attribute so we can get the cpu for
     // the function.
     if (TD) {
       ParsedTargetAttr ParsedAttr = TD->parse();
       if (!ParsedAttr.Architecture.empty() &&
           getTarget().isValidCPUName(ParsedAttr.Architecture)) {
         TargetCPU = ParsedAttr.Architecture;
         TuneCPU = ""; // Clear the tune CPU.
       }
       if (!ParsedAttr.Tune.empty() &&
           getTarget().isValidCPUName(ParsedAttr.Tune))
         TuneCPU = ParsedAttr.Tune;
     }
   } else {
     // Otherwise just add the existing target cpu and target features to the
     // function.
     Features = getTarget().getTargetOpts().Features;
   }
 
   if (!TargetCPU.empty()) {
     Attrs.addAttribute("target-cpu", TargetCPU);
     AddedAttr = true;
   }
   if (!TuneCPU.empty()) {
     Attrs.addAttribute("tune-cpu", TuneCPU);
     AddedAttr = true;
   }
   if (!Features.empty()) {
     llvm::sort(Features);
     Attrs.addAttribute("target-features", llvm::join(Features, ","));
     AddedAttr = true;
   }
 
   return AddedAttr;
 }
 
 void CodeGenModule::setNonAliasAttributes(GlobalDecl GD,
                                           llvm::GlobalObject *GO) {
   const Decl *D = GD.getDecl();
   SetCommonAttributes(GD, GO);
 
   if (D) {
     if (auto *GV = dyn_cast<llvm::GlobalVariable>(GO)) {
       if (D->hasAttr<RetainAttr>())
         addUsedGlobal(GV);
       if (auto *SA = D->getAttr<PragmaClangBSSSectionAttr>())
         GV->addAttribute("bss-section", SA->getName());
       if (auto *SA = D->getAttr<PragmaClangDataSectionAttr>())
         GV->addAttribute("data-section", SA->getName());
       if (auto *SA = D->getAttr<PragmaClangRodataSectionAttr>())
         GV->addAttribute("rodata-section", SA->getName());
       if (auto *SA = D->getAttr<PragmaClangRelroSectionAttr>())
         GV->addAttribute("relro-section", SA->getName());
     }
 
     if (auto *F = dyn_cast<llvm::Function>(GO)) {
       if (D->hasAttr<RetainAttr>())
         addUsedGlobal(F);
       if (auto *SA = D->getAttr<PragmaClangTextSectionAttr>())
         if (!D->getAttr<SectionAttr>())
           F->addFnAttr("implicit-section-name", SA->getName());
 
       llvm::AttrBuilder Attrs(F->getContext());
       if (GetCPUAndFeaturesAttributes(GD, Attrs)) {
         // We know that GetCPUAndFeaturesAttributes will always have the
         // newest set, since it has the newest possible FunctionDecl, so the
         // new ones should replace the old.
         llvm::AttributeMask RemoveAttrs;
         RemoveAttrs.addAttribute("target-cpu");
         RemoveAttrs.addAttribute("target-features");
         RemoveAttrs.addAttribute("tune-cpu");
         F->removeFnAttrs(RemoveAttrs);
         F->addFnAttrs(Attrs);
       }
     }
 
     if (const auto *CSA = D->getAttr<CodeSegAttr>())
       GO->setSection(CSA->getName());
     else if (const auto *SA = D->getAttr<SectionAttr>())
       GO->setSection(SA->getName());
   }
 
   getTargetCodeGenInfo().setTargetAttributes(D, GO, *this);
 }
 
 void CodeGenModule::SetInternalFunctionAttributes(GlobalDecl GD,
                                                   llvm::Function *F,
                                                   const CGFunctionInfo &FI) {
   const Decl *D = GD.getDecl();
   SetLLVMFunctionAttributes(GD, FI, F, /*IsThunk=*/false);
   SetLLVMFunctionAttributesForDefinition(D, F);
 
   F->setLinkage(llvm::Function::InternalLinkage);
 
   setNonAliasAttributes(GD, F);
 }
 
 static void setLinkageForGV(llvm::GlobalValue *GV, const NamedDecl *ND) {
   // Set linkage and visibility in case we never see a definition.
   LinkageInfo LV = ND->getLinkageAndVisibility();
   // Don't set internal linkage on declarations.
   // "extern_weak" is overloaded in LLVM; we probably should have
   // separate linkage types for this.
   if (isExternallyVisible(LV.getLinkage()) &&
       (ND->hasAttr<WeakAttr>() || ND->isWeakImported()))
     GV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
 }
 
 void CodeGenModule::CreateFunctionTypeMetadataForIcall(const FunctionDecl *FD,
                                                        llvm::Function *F) {
   // Only if we are checking indirect calls.
   if (!LangOpts.Sanitize.has(SanitizerKind::CFIICall))
     return;
 
   // Non-static class methods are handled via vtable or member function pointer
   // checks elsewhere.
   if (isa<CXXMethodDecl>(FD) && !cast<CXXMethodDecl>(FD)->isStatic())
     return;
 
   llvm::Metadata *MD = CreateMetadataIdentifierForType(FD->getType());
   F->addTypeMetadata(0, MD);
   F->addTypeMetadata(0, CreateMetadataIdentifierGeneralized(FD->getType()));
 
   // Emit a hash-based bit set entry for cross-DSO calls.
   if (CodeGenOpts.SanitizeCfiCrossDso)
     if (auto CrossDsoTypeId = CreateCrossDsoCfiTypeId(MD))
       F->addTypeMetadata(0, llvm::ConstantAsMetadata::get(CrossDsoTypeId));
 }
 
 void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F,
                                           bool IsIncompleteFunction,
                                           bool IsThunk) {
 
   if (llvm::Intrinsic::ID IID = F->getIntrinsicID()) {
     // If this is an intrinsic function, set the function's attributes
     // to the intrinsic's attributes.
     F->setAttributes(llvm::Intrinsic::getAttributes(getLLVMContext(), IID));
     return;
   }
 
   const auto *FD = cast<FunctionDecl>(GD.getDecl());
 
   if (!IsIncompleteFunction)
     SetLLVMFunctionAttributes(GD, getTypes().arrangeGlobalDeclaration(GD), F,
                               IsThunk);
 
   // Add the Returned attribute for "this", except for iOS 5 and earlier
   // where substantial code, including the libstdc++ dylib, was compiled with
   // GCC and does not actually return "this".
   if (!IsThunk && getCXXABI().HasThisReturn(GD) &&
       !(getTriple().isiOS() && getTriple().isOSVersionLT(6))) {
     assert(!F->arg_empty() &&
            F->arg_begin()->getType()
              ->canLosslesslyBitCastTo(F->getReturnType()) &&
            "unexpected this return");
     F->addParamAttr(0, llvm::Attribute::Returned);
   }
 
   // Only a few attributes are set on declarations; these may later be
   // overridden by a definition.
 
   setLinkageForGV(F, FD);
   setGVProperties(F, FD);
 
   // Setup target-specific attributes.
   if (!IsIncompleteFunction && F->isDeclaration())
     getTargetCodeGenInfo().setTargetAttributes(FD, F, *this);
 
   if (const auto *CSA = FD->getAttr<CodeSegAttr>())
     F->setSection(CSA->getName());
   else if (const auto *SA = FD->getAttr<SectionAttr>())
      F->setSection(SA->getName());
 
   if (const auto *EA = FD->getAttr<ErrorAttr>()) {
     if (EA->isError())
       F->addFnAttr("dontcall-error", EA->getUserDiagnostic());
     else if (EA->isWarning())
       F->addFnAttr("dontcall-warn", EA->getUserDiagnostic());
   }
 
   // If we plan on emitting this inline builtin, we can't treat it as a builtin.
   if (FD->isInlineBuiltinDeclaration()) {
     const FunctionDecl *FDBody;
     bool HasBody = FD->hasBody(FDBody);
     (void)HasBody;
     assert(HasBody && "Inline builtin declarations should always have an "
                       "available body!");
     if (shouldEmitFunction(FDBody))
       F->addFnAttr(llvm::Attribute::NoBuiltin);
   }
 
   if (FD->isReplaceableGlobalAllocationFunction()) {
     // A replaceable global allocation function does not act like a builtin by
     // default, only if it is invoked by a new-expression or delete-expression.
     F->addFnAttr(llvm::Attribute::NoBuiltin);
   }
 
   if (isa<CXXConstructorDecl>(FD) || isa<CXXDestructorDecl>(FD))
     F->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
   else if (const auto *MD = dyn_cast<CXXMethodDecl>(FD))
     if (MD->isVirtual())
       F->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
 
   // Don't emit entries for function declarations in the cross-DSO mode. This
   // is handled with better precision by the receiving DSO. But if jump tables
   // are non-canonical then we need type metadata in order to produce the local
   // jump table.
   if (!CodeGenOpts.SanitizeCfiCrossDso ||
       !CodeGenOpts.SanitizeCfiCanonicalJumpTables)
     CreateFunctionTypeMetadataForIcall(FD, F);
 
   if (getLangOpts().OpenMP && FD->hasAttr<OMPDeclareSimdDeclAttr>())
     getOpenMPRuntime().emitDeclareSimdFunction(FD, F);
 
   if (const auto *CB = FD->getAttr<CallbackAttr>()) {
     // Annotate the callback behavior as metadata:
     //  - The callback callee (as argument number).
     //  - The callback payloads (as argument numbers).
     llvm::LLVMContext &Ctx = F->getContext();
     llvm::MDBuilder MDB(Ctx);
 
     // The payload indices are all but the first one in the encoding. The first
     // identifies the callback callee.
     int CalleeIdx = *CB->encoding_begin();
     ArrayRef<int> PayloadIndices(CB->encoding_begin() + 1, CB->encoding_end());
     F->addMetadata(llvm::LLVMContext::MD_callback,
                    *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
                                                CalleeIdx, PayloadIndices,
                                                /* VarArgsArePassed */ false)}));
   }
 }
 
 void CodeGenModule::addUsedGlobal(llvm::GlobalValue *GV) {
   assert((isa<llvm::Function>(GV) || !GV->isDeclaration()) &&
          "Only globals with definition can force usage.");
   LLVMUsed.emplace_back(GV);
 }
 
 void CodeGenModule::addCompilerUsedGlobal(llvm::GlobalValue *GV) {
   assert(!GV->isDeclaration() &&
          "Only globals with definition can force usage.");
   LLVMCompilerUsed.emplace_back(GV);
 }
 
 void CodeGenModule::addUsedOrCompilerUsedGlobal(llvm::GlobalValue *GV) {
   assert((isa<llvm::Function>(GV) || !GV->isDeclaration()) &&
          "Only globals with definition can force usage.");
   if (getTriple().isOSBinFormatELF())
     LLVMCompilerUsed.emplace_back(GV);
   else
     LLVMUsed.emplace_back(GV);
 }
 
 static void emitUsed(CodeGenModule &CGM, StringRef Name,
                      std::vector<llvm::WeakTrackingVH> &List) {
   // Don't create llvm.used if there is no need.
   if (List.empty())
     return;
 
   // Convert List to what ConstantArray needs.
   SmallVector<llvm::Constant*, 8> UsedArray;
   UsedArray.resize(List.size());
   for (unsigned i = 0, e = List.size(); i != e; ++i) {
     UsedArray[i] =
         llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
             cast<llvm::Constant>(&*List[i]), CGM.Int8PtrTy);
   }
 
   if (UsedArray.empty())
     return;
   llvm::ArrayType *ATy = llvm::ArrayType::get(CGM.Int8PtrTy, UsedArray.size());
 
   auto *GV = new llvm::GlobalVariable(
       CGM.getModule(), ATy, false, llvm::GlobalValue::AppendingLinkage,
       llvm::ConstantArray::get(ATy, UsedArray), Name);
 
   GV->setSection("llvm.metadata");
 }
 
 void CodeGenModule::emitLLVMUsed() {
   emitUsed(*this, "llvm.used", LLVMUsed);
   emitUsed(*this, "llvm.compiler.used", LLVMCompilerUsed);
 }
 
 void CodeGenModule::AppendLinkerOptions(StringRef Opts) {
   auto *MDOpts = llvm::MDString::get(getLLVMContext(), Opts);
   LinkerOptionsMetadata.push_back(llvm::MDNode::get(getLLVMContext(), MDOpts));
 }
 
 void CodeGenModule::AddDetectMismatch(StringRef Name, StringRef Value) {
   llvm::SmallString<32> Opt;
   getTargetCodeGenInfo().getDetectMismatchOption(Name, Value, Opt);
   if (Opt.empty())
     return;
   auto *MDOpts = llvm::MDString::get(getLLVMContext(), Opt);
   LinkerOptionsMetadata.push_back(llvm::MDNode::get(getLLVMContext(), MDOpts));
 }
 
 void CodeGenModule::AddDependentLib(StringRef Lib) {
   auto &C = getLLVMContext();
   if (getTarget().getTriple().isOSBinFormatELF()) {
       ELFDependentLibraries.push_back(
         llvm::MDNode::get(C, llvm::MDString::get(C, Lib)));
     return;
   }
 
   llvm::SmallString<24> Opt;
   getTargetCodeGenInfo().getDependentLibraryOption(Lib, Opt);
   auto *MDOpts = llvm::MDString::get(getLLVMContext(), Opt);
   LinkerOptionsMetadata.push_back(llvm::MDNode::get(C, MDOpts));
 }
 
 /// Add link options implied by the given module, including modules
 /// it depends on, using a postorder walk.
 static void addLinkOptionsPostorder(CodeGenModule &CGM, Module *Mod,
                                     SmallVectorImpl<llvm::MDNode *> &Metadata,
                                     llvm::SmallPtrSet<Module *, 16> &Visited) {
   // Import this module's parent.
   if (Mod->Parent && Visited.insert(Mod->Parent).second) {
     addLinkOptionsPostorder(CGM, Mod->Parent, Metadata, Visited);
   }
 
   // Import this module's dependencies.
   for (Module *Import : llvm::reverse(Mod->Imports)) {
     if (Visited.insert(Import).second)
       addLinkOptionsPostorder(CGM, Import, Metadata, Visited);
   }
 
   // Add linker options to link against the libraries/frameworks
   // described by this module.
   llvm::LLVMContext &Context = CGM.getLLVMContext();
   bool IsELF = CGM.getTarget().getTriple().isOSBinFormatELF();
 
   // For modules that use export_as for linking, use that module
   // name instead.
   if (Mod->UseExportAsModuleLinkName)
     return;
 
   for (const Module::LinkLibrary &LL : llvm::reverse(Mod->LinkLibraries)) {
     // Link against a framework.  Frameworks are currently Darwin only, so we
     // don't to ask TargetCodeGenInfo for the spelling of the linker option.
     if (LL.IsFramework) {
       llvm::Metadata *Args[2] = {llvm::MDString::get(Context, "-framework"),
                                  llvm::MDString::get(Context, LL.Library)};
 
       Metadata.push_back(llvm::MDNode::get(Context, Args));
       continue;
     }
 
     // Link against a library.
     if (IsELF) {
       llvm::Metadata *Args[2] = {
           llvm::MDString::get(Context, "lib"),
           llvm::MDString::get(Context, LL.Library),
       };
       Metadata.push_back(llvm::MDNode::get(Context, Args));
     } else {
       llvm::SmallString<24> Opt;
       CGM.getTargetCodeGenInfo().getDependentLibraryOption(LL.Library, Opt);
       auto *OptString = llvm::MDString::get(Context, Opt);
       Metadata.push_back(llvm::MDNode::get(Context, OptString));
     }
   }
 }
 
 void CodeGenModule::EmitModuleLinkOptions() {
   // Collect the set of all of the modules we want to visit to emit link
   // options, which is essentially the imported modules and all of their
   // non-explicit child modules.
   llvm::SetVector<clang::Module *> LinkModules;
   llvm::SmallPtrSet<clang::Module *, 16> Visited;
   SmallVector<clang::Module *, 16> Stack;
 
   // Seed the stack with imported modules.
   for (Module *M : ImportedModules) {
     // Do not add any link flags when an implementation TU of a module imports
     // a header of that same module.
     if (M->getTopLevelModuleName() == getLangOpts().CurrentModule &&
         !getLangOpts().isCompilingModule())
       continue;
     if (Visited.insert(M).second)
       Stack.push_back(M);
   }
 
   // Find all of the modules to import, making a little effort to prune
   // non-leaf modules.
   while (!Stack.empty()) {
     clang::Module *Mod = Stack.pop_back_val();
 
     bool AnyChildren = false;
 
     // Visit the submodules of this module.
     for (const auto &SM : Mod->submodules()) {
       // Skip explicit children; they need to be explicitly imported to be
       // linked against.
       if (SM->IsExplicit)
         continue;
 
       if (Visited.insert(SM).second) {
         Stack.push_back(SM);
         AnyChildren = true;
       }
     }
 
     // We didn't find any children, so add this module to the list of
     // modules to link against.
     if (!AnyChildren) {
       LinkModules.insert(Mod);
     }
   }
 
   // Add link options for all of the imported modules in reverse topological
   // order.  We don't do anything to try to order import link flags with respect
   // to linker options inserted by things like #pragma comment().
   SmallVector<llvm::MDNode *, 16> MetadataArgs;
   Visited.clear();
   for (Module *M : LinkModules)
     if (Visited.insert(M).second)
       addLinkOptionsPostorder(*this, M, MetadataArgs, Visited);
   std::reverse(MetadataArgs.begin(), MetadataArgs.end());
   LinkerOptionsMetadata.append(MetadataArgs.begin(), MetadataArgs.end());
 
   // Add the linker options metadata flag.
   auto *NMD = getModule().getOrInsertNamedMetadata("llvm.linker.options");
   for (auto *MD : LinkerOptionsMetadata)
     NMD->addOperand(MD);
 }
 
 void CodeGenModule::EmitDeferred() {
   // Emit deferred declare target declarations.
   if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd)
     getOpenMPRuntime().emitDeferredTargetDecls();
 
   // Emit code for any potentially referenced deferred decls.  Since a
   // previously unused static decl may become used during the generation of code
   // for a static function, iterate until no changes are made.
 
   if (!DeferredVTables.empty()) {
     EmitDeferredVTables();
 
     // Emitting a vtable doesn't directly cause more vtables to
     // become deferred, although it can cause functions to be
     // emitted that then need those vtables.
     assert(DeferredVTables.empty());
   }
 
   // Emit CUDA/HIP static device variables referenced by host code only.
   // Note we should not clear CUDADeviceVarODRUsedByHost since it is still
   // needed for further handling.
   if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice)
     for (const auto *V : getContext().CUDADeviceVarODRUsedByHost)
       DeferredDeclsToEmit.push_back(V);
 
   // Stop if we're out of both deferred vtables and deferred declarations.
   if (DeferredDeclsToEmit.empty())
     return;
 
   // Grab the list of decls to emit. If EmitGlobalDefinition schedules more
   // work, it will not interfere with this.
   std::vector<GlobalDecl> CurDeclsToEmit;
   CurDeclsToEmit.swap(DeferredDeclsToEmit);
 
   for (GlobalDecl &D : CurDeclsToEmit) {
     // We should call GetAddrOfGlobal with IsForDefinition set to true in order
     // to get GlobalValue with exactly the type we need, not something that
     // might had been created for another decl with the same mangled name but
     // different type.
     llvm::GlobalValue *GV = dyn_cast<llvm::GlobalValue>(
         GetAddrOfGlobal(D, ForDefinition));
 
     // In case of different address spaces, we may still get a cast, even with
     // IsForDefinition equal to true. Query mangled names table to get
     // GlobalValue.
     if (!GV)
       GV = GetGlobalValue(getMangledName(D));
 
     // Make sure GetGlobalValue returned non-null.
     assert(GV);
 
     // Check to see if we've already emitted this.  This is necessary
     // for a couple of reasons: first, decls can end up in the
     // deferred-decls queue multiple times, and second, decls can end
     // up with definitions in unusual ways (e.g. by an extern inline
     // function acquiring a strong function redefinition).  Just
     // ignore these cases.
     if (!GV->isDeclaration())
       continue;
 
     // If this is OpenMP, check if it is legal to emit this global normally.
     if (LangOpts.OpenMP && OpenMPRuntime && OpenMPRuntime->emitTargetGlobal(D))
       continue;
 
     // Otherwise, emit the definition and move on to the next one.
     EmitGlobalDefinition(D, GV);
 
     // If we found out that we need to emit more decls, do that recursively.
     // This has the advantage that the decls are emitted in a DFS and related
     // ones are close together, which is convenient for testing.
     if (!DeferredVTables.empty() || !DeferredDeclsToEmit.empty()) {
       EmitDeferred();
       assert(DeferredVTables.empty() && DeferredDeclsToEmit.empty());
     }
   }
 }
 
 void CodeGenModule::EmitVTablesOpportunistically() {
   // Try to emit external vtables as available_externally if they have emitted
   // all inlined virtual functions.  It runs after EmitDeferred() and therefore
   // is not allowed to create new references to things that need to be emitted
   // lazily. Note that it also uses fact that we eagerly emitting RTTI.
 
   assert((OpportunisticVTables.empty() || shouldOpportunisticallyEmitVTables())
          && "Only emit opportunistic vtables with optimizations");
 
   for (const CXXRecordDecl *RD : OpportunisticVTables) {
     assert(getVTables().isVTableExternal(RD) &&
            "This queue should only contain external vtables");
     if (getCXXABI().canSpeculativelyEmitVTable(RD))
       VTables.GenerateClassData(RD);
   }
   OpportunisticVTables.clear();
 }
 
 void CodeGenModule::EmitGlobalAnnotations() {
   if (Annotations.empty())
     return;
 
   // Create a new global variable for the ConstantStruct in the Module.
   llvm::Constant *Array = llvm::ConstantArray::get(llvm::ArrayType::get(
     Annotations[0]->getType(), Annotations.size()), Annotations);
   auto *gv = new llvm::GlobalVariable(getModule(), Array->getType(), false,
                                       llvm::GlobalValue::AppendingLinkage,
                                       Array, "llvm.global.annotations");
   gv->setSection(AnnotationSection);
 }
 
 llvm::Constant *CodeGenModule::EmitAnnotationString(StringRef Str) {
   llvm::Constant *&AStr = AnnotationStrings[Str];
   if (AStr)
     return AStr;
 
   // Not found yet, create a new global.
   llvm::Constant *s = llvm::ConstantDataArray::getString(getLLVMContext(), Str);
   auto *gv =
       new llvm::GlobalVariable(getModule(), s->getType(), true,
                                llvm::GlobalValue::PrivateLinkage, s, ".str");
   gv->setSection(AnnotationSection);
   gv->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
   AStr = gv;
   return gv;
 }
 
 llvm::Constant *CodeGenModule::EmitAnnotationUnit(SourceLocation Loc) {
   SourceManager &SM = getContext().getSourceManager();
   PresumedLoc PLoc = SM.getPresumedLoc(Loc);
   if (PLoc.isValid())
     return EmitAnnotationString(PLoc.getFilename());
   return EmitAnnotationString(SM.getBufferName(Loc));
 }
 
 llvm::Constant *CodeGenModule::EmitAnnotationLineNo(SourceLocation L) {
   SourceManager &SM = getContext().getSourceManager();
   PresumedLoc PLoc = SM.getPresumedLoc(L);
   unsigned LineNo = PLoc.isValid() ? PLoc.getLine() :
     SM.getExpansionLineNumber(L);
   return llvm::ConstantInt::get(Int32Ty, LineNo);
 }
 
 llvm::Constant *CodeGenModule::EmitAnnotationArgs(const AnnotateAttr *Attr) {
   ArrayRef<Expr *> Exprs = {Attr->args_begin(), Attr->args_size()};
   if (Exprs.empty())
     return llvm::ConstantPointerNull::get(GlobalsInt8PtrTy);
 
   llvm::FoldingSetNodeID ID;
   for (Expr *E : Exprs) {
     ID.Add(cast<clang::ConstantExpr>(E)->getAPValueResult());
   }
   llvm::Constant *&Lookup = AnnotationArgs[ID.ComputeHash()];
   if (Lookup)
     return Lookup;
 
   llvm::SmallVector<llvm::Constant *, 4> LLVMArgs;
   LLVMArgs.reserve(Exprs.size());
   ConstantEmitter ConstEmiter(*this);
   llvm::transform(Exprs, std::back_inserter(LLVMArgs), [&](const Expr *E) {
     const auto *CE = cast<clang::ConstantExpr>(E);
     return ConstEmiter.emitAbstract(CE->getBeginLoc(), CE->getAPValueResult(),
                                     CE->getType());
   });
   auto *Struct = llvm::ConstantStruct::getAnon(LLVMArgs);
   auto *GV = new llvm::GlobalVariable(getModule(), Struct->getType(), true,
                                       llvm::GlobalValue::PrivateLinkage, Struct,
                                       ".args");
   GV->setSection(AnnotationSection);
   GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
   auto *Bitcasted = llvm::ConstantExpr::getBitCast(GV, GlobalsInt8PtrTy);
 
   Lookup = Bitcasted;
   return Bitcasted;
 }
 
 llvm::Constant *CodeGenModule::EmitAnnotateAttr(llvm::GlobalValue *GV,
                                                 const AnnotateAttr *AA,
                                                 SourceLocation L) {
   // Get the globals for file name, annotation, and the line number.
   llvm::Constant *AnnoGV = EmitAnnotationString(AA->getAnnotation()),
                  *UnitGV = EmitAnnotationUnit(L),
                  *LineNoCst = EmitAnnotationLineNo(L),
                  *Args = EmitAnnotationArgs(AA);
 
   llvm::Constant *GVInGlobalsAS = GV;
   if (GV->getAddressSpace() !=
       getDataLayout().getDefaultGlobalsAddressSpace()) {
     GVInGlobalsAS = llvm::ConstantExpr::getAddrSpaceCast(
         GV, GV->getValueType()->getPointerTo(
                 getDataLayout().getDefaultGlobalsAddressSpace()));
   }
 
   // Create the ConstantStruct for the global annotation.
   llvm::Constant *Fields[] = {
       llvm::ConstantExpr::getBitCast(GVInGlobalsAS, GlobalsInt8PtrTy),
       llvm::ConstantExpr::getBitCast(AnnoGV, GlobalsInt8PtrTy),
       llvm::ConstantExpr::getBitCast(UnitGV, GlobalsInt8PtrTy),
       LineNoCst,
       Args,
   };
   return llvm::ConstantStruct::getAnon(Fields);
 }
 
 void CodeGenModule::AddGlobalAnnotations(const ValueDecl *D,
                                          llvm::GlobalValue *GV) {
   assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
   // Get the struct elements for these annotations.
   for (const auto *I : D->specific_attrs<AnnotateAttr>())
     Annotations.push_back(EmitAnnotateAttr(GV, I, D->getLocation()));
 }
 
 bool CodeGenModule::isInNoSanitizeList(SanitizerMask Kind, llvm::Function *Fn,
                                        SourceLocation Loc) const {
   const auto &NoSanitizeL = getContext().getNoSanitizeList();
   // NoSanitize by function name.
   if (NoSanitizeL.containsFunction(Kind, Fn->getName()))
     return true;
   // NoSanitize by location.
   if (Loc.isValid())
     return NoSanitizeL.containsLocation(Kind, Loc);
   // If location is unknown, this may be a compiler-generated function. Assume
   // it's located in the main file.
   auto &SM = Context.getSourceManager();
   if (const auto *MainFile = SM.getFileEntryForID(SM.getMainFileID())) {
     return NoSanitizeL.containsFile(Kind, MainFile->getName());
   }
   return false;
 }
 
 bool CodeGenModule::isInNoSanitizeList(llvm::GlobalVariable *GV,
                                        SourceLocation Loc, QualType Ty,
                                        StringRef Category) const {
   // For now globals can be ignored only in ASan and KASan.
   const SanitizerMask EnabledAsanMask =
       LangOpts.Sanitize.Mask &
       (SanitizerKind::Address | SanitizerKind::KernelAddress |
        SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress |
        SanitizerKind::MemTag);
   if (!EnabledAsanMask)
     return false;
   const auto &NoSanitizeL = getContext().getNoSanitizeList();
   if (NoSanitizeL.containsGlobal(EnabledAsanMask, GV->getName(), Category))
     return true;
   if (NoSanitizeL.containsLocation(EnabledAsanMask, Loc, Category))
     return true;
   // Check global type.
   if (!Ty.isNull()) {
     // Drill down the array types: if global variable of a fixed type is
     // not sanitized, we also don't instrument arrays of them.
     while (auto AT = dyn_cast<ArrayType>(Ty.getTypePtr()))
       Ty = AT->getElementType();
     Ty = Ty.getCanonicalType().getUnqualifiedType();
     // Only record types (classes, structs etc.) are ignored.
     if (Ty->isRecordType()) {
       std::string TypeStr = Ty.getAsString(getContext().getPrintingPolicy());
       if (NoSanitizeL.containsType(EnabledAsanMask, TypeStr, Category))
         return true;
     }
   }
   return false;
 }
 
 bool CodeGenModule::imbueXRayAttrs(llvm::Function *Fn, SourceLocation Loc,
                                    StringRef Category) const {
   const auto &XRayFilter = getContext().getXRayFilter();
   using ImbueAttr = XRayFunctionFilter::ImbueAttribute;
   auto Attr = ImbueAttr::NONE;
   if (Loc.isValid())
     Attr = XRayFilter.shouldImbueLocation(Loc, Category);
   if (Attr == ImbueAttr::NONE)
     Attr = XRayFilter.shouldImbueFunction(Fn->getName());
   switch (Attr) {
   case ImbueAttr::NONE:
     return false;
   case ImbueAttr::ALWAYS:
     Fn->addFnAttr("function-instrument", "xray-always");
     break;
   case ImbueAttr::ALWAYS_ARG1:
     Fn->addFnAttr("function-instrument", "xray-always");
     Fn->addFnAttr("xray-log-args", "1");
     break;
   case ImbueAttr::NEVER:
     Fn->addFnAttr("function-instrument", "xray-never");
     break;
   }
   return true;
 }
 
 bool CodeGenModule::isProfileInstrExcluded(llvm::Function *Fn,
                                            SourceLocation Loc) const {
   const auto &ProfileList = getContext().getProfileList();
   // If the profile list is empty, then instrument everything.
   if (ProfileList.isEmpty())
     return false;
   CodeGenOptions::ProfileInstrKind Kind = getCodeGenOpts().getProfileInstr();
   // First, check the function name.
   Optional<bool> V = ProfileList.isFunctionExcluded(Fn->getName(), Kind);
   if (V.hasValue())
     return *V;
   // Next, check the source location.
   if (Loc.isValid()) {
     Optional<bool> V = ProfileList.isLocationExcluded(Loc, Kind);
     if (V.hasValue())
       return *V;
   }
   // If location is unknown, this may be a compiler-generated function. Assume
   // it's located in the main file.
   auto &SM = Context.getSourceManager();
   if (const auto *MainFile = SM.getFileEntryForID(SM.getMainFileID())) {
     Optional<bool> V = ProfileList.isFileExcluded(MainFile->getName(), Kind);
     if (V.hasValue())
       return *V;
   }
   return ProfileList.getDefault();
 }
 
 bool CodeGenModule::MustBeEmitted(const ValueDecl *Global) {
   // Never defer when EmitAllDecls is specified.
   if (LangOpts.EmitAllDecls)
     return true;
 
   if (CodeGenOpts.KeepStaticConsts) {
     const auto *VD = dyn_cast<VarDecl>(Global);
     if (VD && VD->getType().isConstQualified() &&
         VD->getStorageDuration() == SD_Static)
       return true;
   }
 
   return getContext().DeclMustBeEmitted(Global);
 }
 
 bool CodeGenModule::MayBeEmittedEagerly(const ValueDecl *Global) {
   // In OpenMP 5.0 variables and function may be marked as
   // device_type(host/nohost) and we should not emit them eagerly unless we sure
   // that they must be emitted on the host/device. To be sure we need to have
   // seen a declare target with an explicit mentioning of the function, we know
   // we have if the level of the declare target attribute is -1. Note that we
   // check somewhere else if we should emit this at all.
   if (LangOpts.OpenMP >= 50 && !LangOpts.OpenMPSimd) {
     llvm::Optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
         OMPDeclareTargetDeclAttr::getActiveAttr(Global);
     if (!ActiveAttr || (*ActiveAttr)->getLevel() != (unsigned)-1)
       return false;
   }
 
   if (const auto *FD = dyn_cast<FunctionDecl>(Global)) {
     if (FD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
       // Implicit template instantiations may change linkage if they are later
       // explicitly instantiated, so they should not be emitted eagerly.
       return false;
   }
   if (const auto *VD = dyn_cast<VarDecl>(Global))
     if (Context.getInlineVariableDefinitionKind(VD) ==
         ASTContext::InlineVariableDefinitionKind::WeakUnknown)
       // A definition of an inline constexpr static data member may change
       // linkage later if it's redeclared outside the class.
       return false;
   // If OpenMP is enabled and threadprivates must be generated like TLS, delay
   // codegen for global variables, because they may be marked as threadprivate.
   if (LangOpts.OpenMP && LangOpts.OpenMPUseTLS &&
       getContext().getTargetInfo().isTLSSupported() && isa<VarDecl>(Global) &&
       !isTypeConstant(Global->getType(), false) &&
       !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(Global))
     return false;
 
   return true;
 }
 
 ConstantAddress CodeGenModule::GetAddrOfMSGuidDecl(const MSGuidDecl *GD) {
   StringRef Name = getMangledName(GD);
 
   // The UUID descriptor should be pointer aligned.
   CharUnits Alignment = CharUnits::fromQuantity(PointerAlignInBytes);
 
   // Look for an existing global.
   if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name))
     return ConstantAddress(GV, GV->getValueType(), Alignment);
 
   ConstantEmitter Emitter(*this);
   llvm::Constant *Init;
 
   APValue &V = GD->getAsAPValue();
   if (!V.isAbsent()) {
     // If possible, emit the APValue version of the initializer. In particular,
     // this gets the type of the constant right.
     Init = Emitter.emitForInitializer(
         GD->getAsAPValue(), GD->getType().getAddressSpace(), GD->getType());
   } else {
     // As a fallback, directly construct the constant.
     // FIXME: This may get padding wrong under esoteric struct layout rules.
     // MSVC appears to create a complete type 'struct __s_GUID' that it
     // presumably uses to represent these constants.
     MSGuidDecl::Parts Parts = GD->getParts();
     llvm::Constant *Fields[4] = {
         llvm::ConstantInt::get(Int32Ty, Parts.Part1),
         llvm::ConstantInt::get(Int16Ty, Parts.Part2),
         llvm::ConstantInt::get(Int16Ty, Parts.Part3),
         llvm::ConstantDataArray::getRaw(
             StringRef(reinterpret_cast<char *>(Parts.Part4And5), 8), 8,
             Int8Ty)};
     Init = llvm::ConstantStruct::getAnon(Fields);
   }
 
   auto *GV = new llvm::GlobalVariable(
       getModule(), Init->getType(),
       /*isConstant=*/true, llvm::GlobalValue::LinkOnceODRLinkage, Init, Name);
   if (supportsCOMDAT())
     GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
   setDSOLocal(GV);
 
   if (!V.isAbsent()) {
     Emitter.finalize(GV);
     return ConstantAddress(GV, GV->getValueType(), Alignment);
   }
 
   llvm::Type *Ty = getTypes().ConvertTypeForMem(GD->getType());
   llvm::Constant *Addr = llvm::ConstantExpr::getBitCast(
       GV, Ty->getPointerTo(GV->getAddressSpace()));
   return ConstantAddress(Addr, Ty, Alignment);
 }
 
 ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject(
     const TemplateParamObjectDecl *TPO) {
   StringRef Name = getMangledName(TPO);
   CharUnits Alignment = getNaturalTypeAlignment(TPO->getType());
 
   if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name))
     return ConstantAddress(GV, GV->getValueType(), Alignment);
 
   ConstantEmitter Emitter(*this);
   llvm::Constant *Init = Emitter.emitForInitializer(
         TPO->getValue(), TPO->getType().getAddressSpace(), TPO->getType());
 
   if (!Init) {
     ErrorUnsupported(TPO, "template parameter object");
     return ConstantAddress::invalid();
   }
 
   auto *GV = new llvm::GlobalVariable(
       getModule(), Init->getType(),
       /*isConstant=*/true, llvm::GlobalValue::LinkOnceODRLinkage, Init, Name);
   if (supportsCOMDAT())
     GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
   Emitter.finalize(GV);
 
   return ConstantAddress(GV, GV->getValueType(), Alignment);
 }
 
 ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) {
   const AliasAttr *AA = VD->getAttr<AliasAttr>();
   assert(AA && "No alias?");
 
   CharUnits Alignment = getContext().getDeclAlign(VD);
   llvm::Type *DeclTy = getTypes().ConvertTypeForMem(VD->getType());
 
   // See if there is already something with the target's name in the module.
   llvm::GlobalValue *Entry = GetGlobalValue(AA->getAliasee());
   if (Entry) {
     unsigned AS = getContext().getTargetAddressSpace(VD->getType());
     auto Ptr = llvm::ConstantExpr::getBitCast(Entry, DeclTy->getPointerTo(AS));
     return ConstantAddress(Ptr, DeclTy, Alignment);
   }
 
   llvm::Constant *Aliasee;
   if (isa<llvm::FunctionType>(DeclTy))
     Aliasee = GetOrCreateLLVMFunction(AA->getAliasee(), DeclTy,
                                       GlobalDecl(cast<FunctionDecl>(VD)),
                                       /*ForVTable=*/false);
   else
     Aliasee = GetOrCreateLLVMGlobal(AA->getAliasee(), DeclTy, LangAS::Default,
                                     nullptr);
 
   auto *F = cast<llvm::GlobalValue>(Aliasee);
   F->setLinkage(llvm::Function::ExternalWeakLinkage);
   WeakRefReferences.insert(F);
 
   return ConstantAddress(Aliasee, DeclTy, Alignment);
 }
 
 void CodeGenModule::EmitGlobal(GlobalDecl GD) {
   const auto *Global = cast<ValueDecl>(GD.getDecl());
 
   // Weak references don't produce any output by themselves.
   if (Global->hasAttr<WeakRefAttr>())
     return;
 
   // If this is an alias definition (which otherwise looks like a declaration)
   // emit it now.
   if (Global->hasAttr<AliasAttr>())
     return EmitAliasDefinition(GD);
 
   // IFunc like an alias whose value is resolved at runtime by calling resolver.
   if (Global->hasAttr<IFuncAttr>())
     return emitIFuncDefinition(GD);
 
   // If this is a cpu_dispatch multiversion function, emit the resolver.
   if (Global->hasAttr<CPUDispatchAttr>())
     return emitCPUDispatchDefinition(GD);
 
   // If this is CUDA, be selective about which declarations we emit.
   if (LangOpts.CUDA) {
     if (LangOpts.CUDAIsDevice) {
       if (!Global->hasAttr<CUDADeviceAttr>() &&
           !Global->hasAttr<CUDAGlobalAttr>() &&
           !Global->hasAttr<CUDAConstantAttr>() &&
           !Global->hasAttr<CUDASharedAttr>() &&
           !Global->getType()->isCUDADeviceBuiltinSurfaceType() &&
           !Global->getType()->isCUDADeviceBuiltinTextureType())
         return;
     } else {
       // We need to emit host-side 'shadows' for all global
       // device-side variables because the CUDA runtime needs their
       // size and host-side address in order to provide access to
       // their device-side incarnations.
 
       // So device-only functions are the only things we skip.
       if (isa<FunctionDecl>(Global) && !Global->hasAttr<CUDAHostAttr>() &&
           Global->hasAttr<CUDADeviceAttr>())
         return;
 
       assert((isa<FunctionDecl>(Global) || isa<VarDecl>(Global)) &&
              "Expected Variable or Function");
     }
   }
 
   if (LangOpts.OpenMP) {
     // If this is OpenMP, check if it is legal to emit this global normally.
     if (OpenMPRuntime && OpenMPRuntime->emitTargetGlobal(GD))
       return;
     if (auto *DRD = dyn_cast<OMPDeclareReductionDecl>(Global)) {
       if (MustBeEmitted(Global))
         EmitOMPDeclareReduction(DRD);
       return;
     } else if (auto *DMD = dyn_cast<OMPDeclareMapperDecl>(Global)) {
       if (MustBeEmitted(Global))
         EmitOMPDeclareMapper(DMD);
       return;
     }
   }
 
   // Ignore declarations, they will be emitted on their first use.
   if (const auto *FD = dyn_cast<FunctionDecl>(Global)) {
     // Forward declarations are emitted lazily on first use.
     if (!FD->doesThisDeclarationHaveABody()) {
       if (!FD->doesDeclarationForceExternallyVisibleDefinition())
         return;
 
       StringRef MangledName = getMangledName(GD);
 
       // Compute the function info and LLVM type.
       const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
       llvm::Type *Ty = getTypes().GetFunctionType(FI);
 
       GetOrCreateLLVMFunction(MangledName, Ty, GD, /*ForVTable=*/false,
                               /*DontDefer=*/false);
       return;
     }
   } else {
     const auto *VD = cast<VarDecl>(Global);
     assert(VD->isFileVarDecl() && "Cannot emit local var decl as global.");
     if (VD->isThisDeclarationADefinition() != VarDecl::Definition &&
         !Context.isMSStaticDataMemberInlineDefinition(VD)) {
       if (LangOpts.OpenMP) {
         // Emit declaration of the must-be-emitted declare target variable.
         if (llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
                 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
           bool UnifiedMemoryEnabled =
               getOpenMPRuntime().hasRequiresUnifiedSharedMemory();
           if (*Res == OMPDeclareTargetDeclAttr::MT_To &&
               !UnifiedMemoryEnabled) {
             (void)GetAddrOfGlobalVar(VD);
           } else {
             assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
                     (*Res == OMPDeclareTargetDeclAttr::MT_To &&
                      UnifiedMemoryEnabled)) &&
                    "Link clause or to clause with unified memory expected.");
             (void)getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
           }
 
           return;
         }
       }
       // If this declaration may have caused an inline variable definition to
       // change linkage, make sure that it's emitted.
       if (Context.getInlineVariableDefinitionKind(VD) ==
           ASTContext::InlineVariableDefinitionKind::Strong)
         GetAddrOfGlobalVar(VD);
       return;
     }
   }
 
   // Defer code generation to first use when possible, e.g. if this is an inline
   // function. If the global must always be emitted, do it eagerly if possible
   // to benefit from cache locality.
   if (MustBeEmitted(Global) && MayBeEmittedEagerly(Global)) {
     // Emit the definition if it can't be deferred.
     EmitGlobalDefinition(GD);
     return;
   }
 
   // If we're deferring emission of a C++ variable with an
   // initializer, remember the order in which it appeared in the file.
   if (getLangOpts().CPlusPlus && isa<VarDecl>(Global) &&
       cast<VarDecl>(Global)->hasInit()) {
     DelayedCXXInitPosition[Global] = CXXGlobalInits.size();
     CXXGlobalInits.push_back(nullptr);
   }
 
   StringRef MangledName = getMangledName(GD);
   if (GetGlobalValue(MangledName) != nullptr) {
     // The value has already been used and should therefore be emitted.
     addDeferredDeclToEmit(GD);
   } else if (MustBeEmitted(Global)) {
     // The value must be emitted, but cannot be emitted eagerly.
     assert(!MayBeEmittedEagerly(Global));
     addDeferredDeclToEmit(GD);
   } else {
     // Otherwise, remember that we saw a deferred decl with this name.  The
     // first use of the mangled name will cause it to move into
     // DeferredDeclsToEmit.
     DeferredDecls[MangledName] = GD;
   }
 }
 
 // Check if T is a class type with a destructor that's not dllimport.
 static bool HasNonDllImportDtor(QualType T) {
   if (const auto *RT = T->getBaseElementTypeUnsafe()->getAs<RecordType>())
     if (CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
       if (RD->getDestructor() && !RD->getDestructor()->hasAttr<DLLImportAttr>())
         return true;
 
   return false;
 }
 
 namespace {
   struct FunctionIsDirectlyRecursive
       : public ConstStmtVisitor<FunctionIsDirectlyRecursive, bool> {
     const StringRef Name;
     const Builtin::Context &BI;
     FunctionIsDirectlyRecursive(StringRef N, const Builtin::Context &C)
         : Name(N), BI(C) {}
 
     bool VisitCallExpr(const CallExpr *E) {
       const FunctionDecl *FD = E->getDirectCallee();
       if (!FD)
         return false;
       AsmLabelAttr *Attr = FD->getAttr<AsmLabelAttr>();
       if (Attr && Name == Attr->getLabel())
         return true;
       unsigned BuiltinID = FD->getBuiltinID();
       if (!BuiltinID || !BI.isLibFunction(BuiltinID))
         return false;
       StringRef BuiltinName = BI.getName(BuiltinID);
       if (BuiltinName.startswith("__builtin_") &&
           Name == BuiltinName.slice(strlen("__builtin_"), StringRef::npos)) {
         return true;
       }
       return false;
     }
 
     bool VisitStmt(const Stmt *S) {
       for (const Stmt *Child : S->children())
         if (Child && this->Visit(Child))
           return true;
       return false;
     }
   };
 
   // Make sure we're not referencing non-imported vars or functions.
   struct DLLImportFunctionVisitor
       : public RecursiveASTVisitor<DLLImportFunctionVisitor> {
     bool SafeToInline = true;
 
     bool shouldVisitImplicitCode() const { return true; }
 
     bool VisitVarDecl(VarDecl *VD) {
       if (VD->getTLSKind()) {
         // A thread-local variable cannot be imported.
         SafeToInline = false;
         return SafeToInline;
       }
 
       // A variable definition might imply a destructor call.
       if (VD->isThisDeclarationADefinition())
         SafeToInline = !HasNonDllImportDtor(VD->getType());
 
       return SafeToInline;
     }
 
     bool VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
       if (const auto *D = E->getTemporary()->getDestructor())
         SafeToInline = D->hasAttr<DLLImportAttr>();
       return SafeToInline;
     }
 
     bool VisitDeclRefExpr(DeclRefExpr *E) {
       ValueDecl *VD = E->getDecl();
       if (isa<FunctionDecl>(VD))
         SafeToInline = VD->hasAttr<DLLImportAttr>();
       else if (VarDecl *V = dyn_cast<VarDecl>(VD))
         SafeToInline = !V->hasGlobalStorage() || V->hasAttr<DLLImportAttr>();
       return SafeToInline;
     }
 
     bool VisitCXXConstructExpr(CXXConstructExpr *E) {
       SafeToInline = E->getConstructor()->hasAttr<DLLImportAttr>();
       return SafeToInline;
     }
 
     bool VisitCXXMemberCallExpr(CXXMemberCallExpr *E) {
       CXXMethodDecl *M = E->getMethodDecl();
       if (!M) {
         // Call through a pointer to member function. This is safe to inline.
         SafeToInline = true;
       } else {
         SafeToInline = M->hasAttr<DLLImportAttr>();
       }
       return SafeToInline;
     }
 
     bool VisitCXXDeleteExpr(CXXDeleteExpr *E) {
       SafeToInline = E->getOperatorDelete()->hasAttr<DLLImportAttr>();
       return SafeToInline;
     }
 
     bool VisitCXXNewExpr(CXXNewExpr *E) {
       SafeToInline = E->getOperatorNew()->hasAttr<DLLImportAttr>();
       return SafeToInline;
     }
   };
 }
 
 // isTriviallyRecursive - Check if this function calls another
 // decl that, because of the asm attribute or the other decl being a builtin,
 // ends up pointing to itself.
 bool
 CodeGenModule::isTriviallyRecursive(const FunctionDecl *FD) {
   StringRef Name;
   if (getCXXABI().getMangleContext().shouldMangleDeclName(FD)) {
     // asm labels are a special kind of mangling we have to support.
     AsmLabelAttr *Attr = FD->getAttr<AsmLabelAttr>();
     if (!Attr)
       return false;
     Name = Attr->getLabel();
   } else {
     Name = FD->getName();
   }
 
   FunctionIsDirectlyRecursive Walker(Name, Context.BuiltinInfo);
   const Stmt *Body = FD->getBody();
   return Body ? Walker.Visit(Body) : false;
 }
 
 bool CodeGenModule::shouldEmitFunction(GlobalDecl GD) {
   if (getFunctionLinkage(GD) != llvm::Function::AvailableExternallyLinkage)
     return true;
   const auto *F = cast<FunctionDecl>(GD.getDecl());
   if (CodeGenOpts.OptimizationLevel == 0 && !F->hasAttr<AlwaysInlineAttr>())
     return false;
 
   if (F->hasAttr<DLLImportAttr>() && !F->hasAttr<AlwaysInlineAttr>()) {
     // Check whether it would be safe to inline this dllimport function.
     DLLImportFunctionVisitor Visitor;
     Visitor.TraverseFunctionDecl(const_cast<FunctionDecl*>(F));
     if (!Visitor.SafeToInline)
       return false;
 
     if (const CXXDestructorDecl *Dtor = dyn_cast<CXXDestructorDecl>(F)) {
       // Implicit destructor invocations aren't captured in the AST, so the
       // check above can't see them. Check for them manually here.
       for (const Decl *Member : Dtor->getParent()->decls())
         if (isa<FieldDecl>(Member))
           if (HasNonDllImportDtor(cast<FieldDecl>(Member)->getType()))
             return false;
       for (const CXXBaseSpecifier &B : Dtor->getParent()->bases())
         if (HasNonDllImportDtor(B.getType()))
           return false;
     }
   }
 
   // Inline builtins declaration must be emitted. They often are fortified
   // functions.
   if (F->isInlineBuiltinDeclaration())
     return true;
 
   // PR9614. Avoid cases where the source code is lying to us. An available
   // externally function should have an equivalent function somewhere else,
   // but a function that calls itself through asm label/`__builtin_` trickery is
   // clearly not equivalent to the real implementation.
   // This happens in glibc's btowc and in some configure checks.
   return !isTriviallyRecursive(F);
 }
 
 bool CodeGenModule::shouldOpportunisticallyEmitVTables() {
   return CodeGenOpts.OptimizationLevel > 0;
 }
 
 void CodeGenModule::EmitMultiVersionFunctionDefinition(GlobalDecl GD,
                                                        llvm::GlobalValue *GV) {
   const auto *FD = cast<FunctionDecl>(GD.getDecl());
 
   if (FD->isCPUSpecificMultiVersion()) {
     auto *Spec = FD->getAttr<CPUSpecificAttr>();
     for (unsigned I = 0; I < Spec->cpus_size(); ++I)
       EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr);
     // Requires multiple emits.
   } else if (FD->isTargetClonesMultiVersion()) {
     auto *Clone = FD->getAttr<TargetClonesAttr>();
     for (unsigned I = 0; I < Clone->featuresStrs_size(); ++I)
       if (Clone->isFirstOfVersion(I))
         EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr);
     EmitTargetClonesResolver(GD);
   } else
     EmitGlobalFunctionDefinition(GD, GV);
 }
 
 void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD, llvm::GlobalValue *GV) {
   const auto *D = cast<ValueDecl>(GD.getDecl());
 
   PrettyStackTraceDecl CrashInfo(const_cast<ValueDecl *>(D), D->getLocation(),
                                  Context.getSourceManager(),
                                  "Generating code for declaration");
 
   if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
     // At -O0, don't generate IR for functions with available_externally
     // linkage.
     if (!shouldEmitFunction(GD))
       return;
 
     llvm::TimeTraceScope TimeScope("CodeGen Function", [&]() {
       std::string Name;
       llvm::raw_string_ostream OS(Name);
       FD->getNameForDiagnostic(OS, getContext().getPrintingPolicy(),
                                /*Qualified=*/true);
       return Name;
     });
 
     if (const auto *Method = dyn_cast<CXXMethodDecl>(D)) {
       // Make sure to emit the definition(s) before we emit the thunks.
       // This is necessary for the generation of certain thunks.
       if (isa<CXXConstructorDecl>(Method) || isa<CXXDestructorDecl>(Method))
         ABI->emitCXXStructor(GD);
       else if (FD->isMultiVersion())
         EmitMultiVersionFunctionDefinition(GD, GV);
       else
         EmitGlobalFunctionDefinition(GD, GV);
 
       if (Method->isVirtual())
         getVTables().EmitThunks(GD);
 
       return;
     }
 
     if (FD->isMultiVersion())
       return EmitMultiVersionFunctionDefinition(GD, GV);
     return EmitGlobalFunctionDefinition(GD, GV);
   }
 
   if (const auto *VD = dyn_cast<VarDecl>(D))
     return EmitGlobalVarDefinition(VD, !VD->hasDefinition());
 
   llvm_unreachable("Invalid argument to EmitGlobalDefinition()");
 }
 
 static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
                                                       llvm::Function *NewFn);
 
 static unsigned
 TargetMVPriority(const TargetInfo &TI,
                  const CodeGenFunction::MultiVersionResolverOption &RO) {
   unsigned Priority = 0;
   for (StringRef Feat : RO.Conditions.Features)
     Priority = std::max(Priority, TI.multiVersionSortPriority(Feat));
 
   if (!RO.Conditions.Architecture.empty())
     Priority = std::max(
         Priority, TI.multiVersionSortPriority(RO.Conditions.Architecture));
   return Priority;
 }
 
 // Multiversion functions should be at most 'WeakODRLinkage' so that a different
 // TU can forward declare the function without causing problems.  Particularly
 // in the cases of CPUDispatch, this causes issues. This also makes sure we
 // work with internal linkage functions, so that the same function name can be
 // used with internal linkage in multiple TUs.
 llvm::GlobalValue::LinkageTypes getMultiversionLinkage(CodeGenModule &CGM,
                                                        GlobalDecl GD) {
   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
   if (FD->getFormalLinkage() == InternalLinkage)
     return llvm::GlobalValue::InternalLinkage;
   return llvm::GlobalValue::WeakODRLinkage;
 }
 
 void CodeGenModule::EmitTargetClonesResolver(GlobalDecl GD) {
   const auto *FD = cast<FunctionDecl>(GD.getDecl());
   assert(FD && "Not a FunctionDecl?");
   const auto *TC = FD->getAttr<TargetClonesAttr>();
   assert(TC && "Not a target_clones Function?");
 
   QualType CanonTy = Context.getCanonicalType(FD->getType());
   llvm::Type *DeclTy = getTypes().ConvertType(CanonTy);
 
   if (const auto *CXXFD = dyn_cast<CXXMethodDecl>(FD)) {
     const CGFunctionInfo &FInfo = getTypes().arrangeCXXMethodDeclaration(CXXFD);
     DeclTy = getTypes().GetFunctionType(FInfo);
   }
 
   llvm::Function *ResolverFunc;
   if (getTarget().supportsIFunc()) {
     auto *IFunc = cast<llvm::GlobalIFunc>(
         GetOrCreateMultiVersionResolver(GD, DeclTy, FD));
     ResolverFunc = cast<llvm::Function>(IFunc->getResolver());
   } else
     ResolverFunc =
         cast<llvm::Function>(GetOrCreateMultiVersionResolver(GD, DeclTy, FD));
 
   SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
   for (unsigned VersionIndex = 0; VersionIndex < TC->featuresStrs_size();
        ++VersionIndex) {
     if (!TC->isFirstOfVersion(VersionIndex))
       continue;
     StringRef Version = TC->getFeatureStr(VersionIndex);
     StringRef MangledName =
         getMangledName(GD.getWithMultiVersionIndex(VersionIndex));
     llvm::Constant *Func = GetGlobalValue(MangledName);
     assert(Func &&
            "Should have already been created before calling resolver emit");
 
     StringRef Architecture;
     llvm::SmallVector<StringRef, 1> Feature;
 
     if (Version.startswith("arch="))
       Architecture = Version.drop_front(sizeof("arch=") - 1);
     else if (Version != "default")
       Feature.push_back(Version);
 
     Options.emplace_back(cast<llvm::Function>(Func), Architecture, Feature);
   }
 
   const TargetInfo &TI = getTarget();
   std::stable_sort(
       Options.begin(), Options.end(),
       [&TI](const CodeGenFunction::MultiVersionResolverOption &LHS,
             const CodeGenFunction::MultiVersionResolverOption &RHS) {
         return TargetMVPriority(TI, LHS) > TargetMVPriority(TI, RHS);
       });
   CodeGenFunction CGF(*this);
   CGF.EmitMultiVersionResolver(ResolverFunc, Options);
 }
 
 void CodeGenModule::emitMultiVersionFunctions() {
   std::vector<GlobalDecl> MVFuncsToEmit;
   MultiVersionFuncs.swap(MVFuncsToEmit);
   for (GlobalDecl GD : MVFuncsToEmit) {
     SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
     const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
     getContext().forEachMultiversionedFunctionVersion(
         FD, [this, &GD, &Options](const FunctionDecl *CurFD) {
           GlobalDecl CurGD{
               (CurFD->isDefined() ? CurFD->getDefinition() : CurFD)};
           StringRef MangledName = getMangledName(CurGD);
           llvm::Constant *Func = GetGlobalValue(MangledName);
           if (!Func) {
             if (CurFD->isDefined()) {
               EmitGlobalFunctionDefinition(CurGD, nullptr);
               Func = GetGlobalValue(MangledName);
             } else {
               const CGFunctionInfo &FI =
                   getTypes().arrangeGlobalDeclaration(GD);
               llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
               Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false,
                                        /*DontDefer=*/false, ForDefinition);
             }
             assert(Func && "This should have just been created");
           }
 
           const auto *TA = CurFD->getAttr<TargetAttr>();
           llvm::SmallVector<StringRef, 8> Feats;
           TA->getAddedFeatures(Feats);
 
           Options.emplace_back(cast<llvm::Function>(Func),
                                TA->getArchitecture(), Feats);
         });
 
     llvm::Function *ResolverFunc;
     const TargetInfo &TI = getTarget();
 
     if (TI.supportsIFunc() || FD->isTargetMultiVersion()) {
       ResolverFunc = cast<llvm::Function>(
           GetGlobalValue((getMangledName(GD) + ".resolver").str()));
       ResolverFunc->setLinkage(getMultiversionLinkage(*this, GD));
     } else {
       ResolverFunc = cast<llvm::Function>(GetGlobalValue(getMangledName(GD)));
     }
 
     if (supportsCOMDAT())
       ResolverFunc->setComdat(
           getModule().getOrInsertComdat(ResolverFunc->getName()));
 
     llvm::stable_sort(
         Options, [&TI](const CodeGenFunction::MultiVersionResolverOption &LHS,
                        const CodeGenFunction::MultiVersionResolverOption &RHS) {
           return TargetMVPriority(TI, LHS) > TargetMVPriority(TI, RHS);
         });
     CodeGenFunction CGF(*this);
     CGF.EmitMultiVersionResolver(ResolverFunc, Options);
   }
 
   // Ensure that any additions to the deferred decls list caused by emitting a
   // variant are emitted.  This can happen when the variant itself is inline and
   // calls a function without linkage.
   if (!MVFuncsToEmit.empty())
     EmitDeferred();
 
   // Ensure that any additions to the multiversion funcs list from either the
   // deferred decls or the multiversion functions themselves are emitted.
   if (!MultiVersionFuncs.empty())
     emitMultiVersionFunctions();
 }
 
 void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
   const auto *FD = cast<FunctionDecl>(GD.getDecl());
   assert(FD && "Not a FunctionDecl?");
   assert(FD->isCPUDispatchMultiVersion() && "Not a multiversion function?");
   const auto *DD = FD->getAttr<CPUDispatchAttr>();
   assert(DD && "Not a cpu_dispatch Function?");
   llvm::Type *DeclTy = getTypes().ConvertType(FD->getType());
 
   if (const auto *CXXFD = dyn_cast<CXXMethodDecl>(FD)) {
     const CGFunctionInfo &FInfo = getTypes().arrangeCXXMethodDeclaration(CXXFD);
     DeclTy = getTypes().GetFunctionType(FInfo);
   }
 
   StringRef ResolverName = getMangledName(GD);
   UpdateMultiVersionNames(GD, FD, ResolverName);
 
   llvm::Type *ResolverType;
   GlobalDecl ResolverGD;
   if (getTarget().supportsIFunc()) {
     ResolverType = llvm::FunctionType::get(
         llvm::PointerType::get(DeclTy,
                                Context.getTargetAddressSpace(FD->getType())),
         false);
   }
   else {
     ResolverType = DeclTy;
     ResolverGD = GD;
   }
 
   auto *ResolverFunc = cast<llvm::Function>(GetOrCreateLLVMFunction(
       ResolverName, ResolverType, ResolverGD, /*ForVTable=*/false));
   ResolverFunc->setLinkage(getMultiversionLinkage(*this, GD));
   if (supportsCOMDAT())
     ResolverFunc->setComdat(
         getModule().getOrInsertComdat(ResolverFunc->getName()));
 
   SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
   const TargetInfo &Target = getTarget();
   unsigned Index = 0;
   for (const IdentifierInfo *II : DD->cpus()) {
     // Get the name of the target function so we can look it up/create it.
     std::string MangledName = getMangledNameImpl(*this, GD, FD, true) +
                               getCPUSpecificMangling(*this, II->getName());
 
     llvm::Constant *Func = GetGlobalValue(MangledName);
 
     if (!Func) {
       GlobalDecl ExistingDecl = Manglings.lookup(MangledName);
       if (ExistingDecl.getDecl() &&
           ExistingDecl.getDecl()->getAsFunction()->isDefined()) {
         EmitGlobalFunctionDefinition(ExistingDecl, nullptr);
         Func = GetGlobalValue(MangledName);
       } else {
         if (!ExistingDecl.getDecl())
           ExistingDecl = GD.getWithMultiVersionIndex(Index);
 
       Func = GetOrCreateLLVMFunction(
           MangledName, DeclTy, ExistingDecl,
           /*ForVTable=*/false, /*DontDefer=*/true,
           /*IsThunk=*/false, llvm::AttributeList(), ForDefinition);
       }
     }
 
     llvm::SmallVector<StringRef, 32> Features;
     Target.getCPUSpecificCPUDispatchFeatures(II->getName(), Features);
     llvm::transform(Features, Features.begin(),
                     [](StringRef Str) { return Str.substr(1); });
     llvm::erase_if(Features, [&Target](StringRef Feat) {
       return !Target.validateCpuSupports(Feat);
     });
     Options.emplace_back(cast<llvm::Function>(Func), StringRef{}, Features);
     ++Index;
   }
 
   llvm::stable_sort(
       Options, [](const CodeGenFunction::MultiVersionResolverOption &LHS,
                   const CodeGenFunction::MultiVersionResolverOption &RHS) {
         return llvm::X86::getCpuSupportsMask(LHS.Conditions.Features) >
                llvm::X86::getCpuSupportsMask(RHS.Conditions.Features);
       });
 
   // If the list contains multiple 'default' versions, such as when it contains
   // 'pentium' and 'generic', don't emit the call to the generic one (since we
   // always run on at least a 'pentium'). We do this by deleting the 'least
   // advanced' (read, lowest mangling letter).
   while (Options.size() > 1 &&
          llvm::X86::getCpuSupportsMask(
              (Options.end() - 2)->Conditions.Features) == 0) {
     StringRef LHSName = (Options.end() - 2)->Function->getName();
     StringRef RHSName = (Options.end() - 1)->Function->getName();
     if (LHSName.compare(RHSName) < 0)
       Options.erase(Options.end() - 2);
     else
       Options.erase(Options.end() - 1);
   }
 
   CodeGenFunction CGF(*this);
   CGF.EmitMultiVersionResolver(ResolverFunc, Options);
 
   if (getTarget().supportsIFunc()) {
     std::string AliasName = getMangledNameImpl(
         *this, GD, FD, /*OmitMultiVersionMangling=*/true);
     llvm::Constant *AliasFunc = GetGlobalValue(AliasName);
     if (!AliasFunc) {
       auto *IFunc = cast<llvm::GlobalIFunc>(GetOrCreateLLVMFunction(
           AliasName, DeclTy, GD, /*ForVTable=*/false, /*DontDefer=*/true,
           /*IsThunk=*/false, llvm::AttributeList(), NotForDefinition));
       auto *GA = llvm::GlobalAlias::create(DeclTy, 0,
                                            getMultiversionLinkage(*this, GD),
                                            AliasName, IFunc, &getModule());
       SetCommonAttributes(GD, GA);
     }
   }
 }
 
 /// If a dispatcher for the specified mangled name is not in the module, create
 /// and return an llvm Function with the specified type.
 llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(
     GlobalDecl GD, llvm::Type *DeclTy, const FunctionDecl *FD) {
   std::string MangledName =
       getMangledNameImpl(*this, GD, FD, /*OmitMultiVersionMangling=*/true);
 
   // Holds the name of the resolver, in ifunc mode this is the ifunc (which has
   // a separate resolver).
   std::string ResolverName = MangledName;
   if (getTarget().supportsIFunc())
     ResolverName += ".ifunc";
   else if (FD->isTargetMultiVersion())
     ResolverName += ".resolver";
 
   // If this already exists, just return that one.
   if (llvm::GlobalValue *ResolverGV = GetGlobalValue(ResolverName))
     return ResolverGV;
 
   // Since this is the first time we've created this IFunc, make sure
   // that we put this multiversioned function into the list to be
   // replaced later if necessary (target multiversioning only).
   if (FD->isTargetMultiVersion())
     MultiVersionFuncs.push_back(GD);
   else if (FD->isTargetClonesMultiVersion()) {
     // In target_clones multiversioning, make sure we emit this if used.
     auto DDI =
         DeferredDecls.find(getMangledName(GD.getWithMultiVersionIndex(0)));
     if (DDI != DeferredDecls.end()) {
       addDeferredDeclToEmit(GD);
       DeferredDecls.erase(DDI);
     } else {
       // Emit the symbol of the 1st variant, so that the deferred decls know we
       // need it, otherwise the only global value will be the resolver/ifunc,
       // which end up getting broken if we search for them with GetGlobalValue'.
       GetOrCreateLLVMFunction(
           getMangledName(GD.getWithMultiVersionIndex(0)), DeclTy, FD,
           /*ForVTable=*/false, /*DontDefer=*/true,
           /*IsThunk=*/false, llvm::AttributeList(), ForDefinition);
     }
   }
 
   if (getTarget().supportsIFunc()) {
     llvm::Type *ResolverType = llvm::FunctionType::get(
         llvm::PointerType::get(
             DeclTy, getContext().getTargetAddressSpace(FD->getType())),
         false);
     llvm::Constant *Resolver = GetOrCreateLLVMFunction(
         MangledName + ".resolver", ResolverType, GlobalDecl{},
         /*ForVTable=*/false);
     llvm::GlobalIFunc *GIF =
         llvm::GlobalIFunc::create(DeclTy, 0, getMultiversionLinkage(*this, GD),
                                   "", Resolver, &getModule());
     GIF->setName(ResolverName);
     SetCommonAttributes(FD, GIF);
 
     return GIF;
   }
 
   llvm::Constant *Resolver = GetOrCreateLLVMFunction(
       ResolverName, DeclTy, GlobalDecl{}, /*ForVTable=*/false);
   assert(isa<llvm::GlobalValue>(Resolver) &&
          "Resolver should be created for the first time");
   SetCommonAttributes(FD, cast<llvm::GlobalValue>(Resolver));
   return Resolver;
 }
 
 /// GetOrCreateLLVMFunction - If the specified mangled name is not in the
 /// module, create and return an llvm Function with the specified type. If there
 /// is something in the module with the specified name, return it potentially
 /// bitcasted to the right type.
 ///
 /// If D is non-null, it specifies a decl that correspond to this.  This is used
 /// to set the attributes on the function when it is first created.
 llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction(
     StringRef MangledName, llvm::Type *Ty, GlobalDecl GD, bool ForVTable,
     bool DontDefer, bool IsThunk, llvm::AttributeList ExtraAttrs,
     ForDefinition_t IsForDefinition) {
   const Decl *D = GD.getDecl();
 
   // Any attempts to use a MultiVersion function should result in retrieving
   // the iFunc instead. Name Mangling will handle the rest of the changes.
   if (const FunctionDecl *FD = cast_or_null<FunctionDecl>(D)) {
     // For the device mark the function as one that should be emitted.
     if (getLangOpts().OpenMPIsDevice && OpenMPRuntime &&
         !OpenMPRuntime->markAsGlobalTarget(GD) && FD->isDefined() &&
         !DontDefer && !IsForDefinition) {
       if (const FunctionDecl *FDDef = FD->getDefinition()) {
         GlobalDecl GDDef;
         if (const auto *CD = dyn_cast<CXXConstructorDecl>(FDDef))
           GDDef = GlobalDecl(CD, GD.getCtorType());
         else if (const auto *DD = dyn_cast<CXXDestructorDecl>(FDDef))
           GDDef = GlobalDecl(DD, GD.getDtorType());
         else
           GDDef = GlobalDecl(FDDef);
         EmitGlobal(GDDef);
       }
     }
 
     if (FD->isMultiVersion()) {
         UpdateMultiVersionNames(GD, FD, MangledName);
       if (!IsForDefinition)
         return GetOrCreateMultiVersionResolver(GD, Ty, FD);
     }
   }
 
   // Lookup the entry, lazily creating it if necessary.
   llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
   if (Entry) {
     if (WeakRefReferences.erase(Entry)) {
       const FunctionDecl *FD = cast_or_null<FunctionDecl>(D);
       if (FD && !FD->hasAttr<WeakAttr>())
         Entry->setLinkage(llvm::Function::ExternalLinkage);
     }
 
     // Handle dropped DLL attributes.
     if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>()) {
       Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
       setDSOLocal(Entry);
     }
 
     // If there are two attempts to define the same mangled name, issue an
     // error.
     if (IsForDefinition && !Entry->isDeclaration()) {
       GlobalDecl OtherGD;
       // Check that GD is not yet in DiagnosedConflictingDefinitions is required
       // to make sure that we issue an error only once.
       if (lookupRepresentativeDecl(MangledName, OtherGD) &&
           (GD.getCanonicalDecl().getDecl() !=
            OtherGD.getCanonicalDecl().getDecl()) &&
           DiagnosedConflictingDefinitions.insert(GD).second) {
         getDiags().Report(D->getLocation(), diag::err_duplicate_mangled_name)
             << MangledName;
         getDiags().Report(OtherGD.getDecl()->getLocation(),
                           diag::note_previous_definition);
       }
     }
 
     if ((isa<llvm::Function>(Entry) || isa<llvm::GlobalAlias>(Entry)) &&
         (Entry->getValueType() == Ty)) {
       return Entry;
     }
 
     // Make sure the result is of the correct type.
     // (If function is requested for a definition, we always need to create a new
     // function, not just return a bitcast.)
     if (!IsForDefinition)
       return llvm::ConstantExpr::getBitCast(Entry, Ty->getPointerTo());
   }
 
   // This function doesn't have a complete type (for example, the return
   // type is an incomplete struct). Use a fake type instead, and make
   // sure not to try to set attributes.
   bool IsIncompleteFunction = false;
 
   llvm::FunctionType *FTy;
   if (isa<llvm::FunctionType>(Ty)) {
     FTy = cast<llvm::FunctionType>(Ty);
   } else {
     FTy = llvm::FunctionType::get(VoidTy, false);
     IsIncompleteFunction = true;
   }
 
   llvm::Function *F =
       llvm::Function::Create(FTy, llvm::Function::ExternalLinkage,
                              Entry ? StringRef() : MangledName, &getModule());
 
   // If we already created a function with the same mangled name (but different
   // type) before, take its name and add it to the list of functions to be
   // replaced with F at the end of CodeGen.
   //
   // This happens if there is a prototype for a function (e.g. "int f()") and
   // then a definition of a different type (e.g. "int f(int x)").
   if (Entry) {
     F->takeName(Entry);
 
     // This might be an implementation of a function without a prototype, in
     // which case, try to do special replacement of calls which match the new
     // prototype.  The really key thing here is that we also potentially drop
     // arguments from the call site so as to make a direct call, which makes the
     // inliner happier and suppresses a number of optimizer warnings (!) about
     // dropping arguments.
     if (!Entry->use_empty()) {
       ReplaceUsesOfNonProtoTypeWithRealFunction(Entry, F);
       Entry->removeDeadConstantUsers();
     }
 
     llvm::Constant *BC = llvm::ConstantExpr::getBitCast(
         F, Entry->getValueType()->getPointerTo());
     addGlobalValReplacement(Entry, BC);
   }
 
   assert(F->getName() == MangledName && "name was uniqued!");
   if (D)
     SetFunctionAttributes(GD, F, IsIncompleteFunction, IsThunk);
   if (ExtraAttrs.hasFnAttrs()) {
     llvm::AttrBuilder B(F->getContext(), ExtraAttrs.getFnAttrs());
     F->addFnAttrs(B);
   }
 
   if (!DontDefer) {
     // All MSVC dtors other than the base dtor are linkonce_odr and delegate to
     // each other bottoming out with the base dtor.  Therefore we emit non-base
     // dtors on usage, even if there is no dtor definition in the TU.
     if (D && isa<CXXDestructorDecl>(D) &&
         getCXXABI().useThunkForDtorVariant(cast<CXXDestructorDecl>(D),
                                            GD.getDtorType()))
       addDeferredDeclToEmit(GD);
 
     // This is the first use or definition of a mangled name.  If there is a
     // deferred decl with this name, remember that we need to emit it at the end
     // of the file.
     auto DDI = DeferredDecls.find(MangledName);
     if (DDI != DeferredDecls.end()) {
       // Move the potentially referenced deferred decl to the
       // DeferredDeclsToEmit list, and remove it from DeferredDecls (since we
       // don't need it anymore).
       addDeferredDeclToEmit(DDI->second);
       DeferredDecls.erase(DDI);
 
       // Otherwise, there are cases we have to worry about where we're
       // using a declaration for which we must emit a definition but where
       // we might not find a top-level definition:
       //   - member functions defined inline in their classes
       //   - friend functions defined inline in some class
       //   - special member functions with implicit definitions
       // If we ever change our AST traversal to walk into class methods,
       // this will be unnecessary.
       //
       // We also don't emit a definition for a function if it's going to be an
       // entry in a vtable, unless it's already marked as used.
     } else if (getLangOpts().CPlusPlus && D) {
       // Look for a declaration that's lexically in a record.
       for (const auto *FD = cast<FunctionDecl>(D)->getMostRecentDecl(); FD;
            FD = FD->getPreviousDecl()) {
         if (isa<CXXRecordDecl>(FD->getLexicalDeclContext())) {
           if (FD->doesThisDeclarationHaveABody()) {
             addDeferredDeclToEmit(GD.getWithDecl(FD));
             break;
           }
         }
       }
     }
   }
 
   // Make sure the result is of the requested type.
   if (!IsIncompleteFunction) {
     assert(F->getFunctionType() == Ty);
     return F;
   }
 
   llvm::Type *PTy = llvm::PointerType::getUnqual(Ty);
   return llvm::ConstantExpr::getBitCast(F, PTy);
 }
 
 /// GetAddrOfFunction - Return the address of the given function.  If Ty is
 /// non-null, then this function will use the specified type if it has to
 /// create it (this occurs when we see a definition of the function).
 llvm::Constant *CodeGenModule::GetAddrOfFunction(GlobalDecl GD,
                                                  llvm::Type *Ty,
                                                  bool ForVTable,
                                                  bool DontDefer,
                                               ForDefinition_t IsForDefinition) {
   assert(!cast<FunctionDecl>(GD.getDecl())->isConsteval() &&
          "consteval function should never be emitted");
   // If there was no specific requested type, just convert it now.
   if (!Ty) {
     const auto *FD = cast<FunctionDecl>(GD.getDecl());
     Ty = getTypes().ConvertType(FD->getType());
   }
 
   // Devirtualized destructor calls may come through here instead of via
   // getAddrOfCXXStructor. Make sure we use the MS ABI base destructor instead
   // of the complete destructor when necessary.
   if (const auto *DD = dyn_cast<CXXDestructorDecl>(GD.getDecl())) {
     if (getTarget().getCXXABI().isMicrosoft() &&
         GD.getDtorType() == Dtor_Complete &&
         DD->getParent()->getNumVBases() == 0)
       GD = GlobalDecl(DD, Dtor_Base);
   }
 
   StringRef MangledName = getMangledName(GD);
   auto *F = GetOrCreateLLVMFunction(MangledName, Ty, GD, ForVTable, DontDefer,
                                     /*IsThunk=*/false, llvm::AttributeList(),
                                     IsForDefinition);
   // Returns kernel handle for HIP kernel stub function.
   if (LangOpts.CUDA && !LangOpts.CUDAIsDevice &&
       cast<FunctionDecl>(GD.getDecl())->hasAttr<CUDAGlobalAttr>()) {
     auto *Handle = getCUDARuntime().getKernelHandle(
         cast<llvm::Function>(F->stripPointerCasts()), GD);
     if (IsForDefinition)
       return F;
     return llvm::ConstantExpr::getBitCast(Handle, Ty->getPointerTo());
   }
   return F;
 }
 
 llvm::Constant *CodeGenModule::GetFunctionStart(const ValueDecl *Decl) {
   llvm::GlobalValue *F =
       cast<llvm::GlobalValue>(GetAddrOfFunction(Decl)->stripPointerCasts());
 
   return llvm::ConstantExpr::getBitCast(llvm::NoCFIValue::get(F),
                                         llvm::Type::getInt8PtrTy(VMContext));
 }
 
 static const FunctionDecl *
 GetRuntimeFunctionDecl(ASTContext &C, StringRef Name) {
   TranslationUnitDecl *TUDecl = C.getTranslationUnitDecl();
   DeclContext *DC = TranslationUnitDecl::castToDeclContext(TUDecl);
 
   IdentifierInfo &CII = C.Idents.get(Name);
   for (const auto *Result : DC->lookup(&CII))
     if (const auto *FD = dyn_cast<FunctionDecl>(Result))
       return FD;
 
   if (!C.getLangOpts().CPlusPlus)
     return nullptr;
 
   // Demangle the premangled name from getTerminateFn()
   IdentifierInfo &CXXII =
       (Name == "_ZSt9terminatev" || Name == "?terminate@@YAXXZ")
           ? C.Idents.get("terminate")
           : C.Idents.get(Name);
 
   for (const auto &N : {"__cxxabiv1", "std"}) {
     IdentifierInfo &NS = C.Idents.get(N);
     for (const auto *Result : DC->lookup(&NS)) {
       const NamespaceDecl *ND = dyn_cast<NamespaceDecl>(Result);
       if (auto *LSD = dyn_cast<LinkageSpecDecl>(Result))
         for (const auto *Result : LSD->lookup(&NS))
           if ((ND = dyn_cast<NamespaceDecl>(Result)))
             break;
 
       if (ND)
         for (const auto *Result : ND->lookup(&CXXII))
           if (const auto *FD = dyn_cast<FunctionDecl>(Result))
             return FD;
     }
   }
 
   return nullptr;
 }
 
 /// CreateRuntimeFunction - Create a new runtime function with the specified
 /// type and name.
 llvm::FunctionCallee
 CodeGenModule::CreateRuntimeFunction(llvm::FunctionType *FTy, StringRef Name,
                                      llvm::AttributeList ExtraAttrs, bool Local,
                                      bool AssumeConvergent) {
   if (AssumeConvergent) {
     ExtraAttrs =
         ExtraAttrs.addFnAttribute(VMContext, llvm::Attribute::Convergent);
   }
 
   llvm::Constant *C =
       GetOrCreateLLVMFunction(Name, FTy, GlobalDecl(), /*ForVTable=*/false,
                               /*DontDefer=*/false, /*IsThunk=*/false,
                               ExtraAttrs);
 
   if (auto *F = dyn_cast<llvm::Function>(C)) {
     if (F->empty()) {
       F->setCallingConv(getRuntimeCC());
 
       // In Windows Itanium environments, try to mark runtime functions
       // dllimport. For Mingw and MSVC, don't. We don't really know if the user
       // will link their standard library statically or dynamically. Marking
       // functions imported when they are not imported can cause linker errors
       // and warnings.
       if (!Local && getTriple().isWindowsItaniumEnvironment() &&
           !getCodeGenOpts().LTOVisibilityPublicStd) {
         const FunctionDecl *FD = GetRuntimeFunctionDecl(Context, Name);
         if (!FD || FD->hasAttr<DLLImportAttr>()) {
           F->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
           F->setLinkage(llvm::GlobalValue::ExternalLinkage);
         }
       }
       setDSOLocal(F);
     }
   }
 
   return {FTy, C};
 }
 
 /// isTypeConstant - Determine whether an object of this type can be emitted
 /// as a constant.
 ///
 /// If ExcludeCtor is true, the duration when the object's constructor runs
 /// will not be considered. The caller will need to verify that the object is
 /// not written to during its construction.
 bool CodeGenModule::isTypeConstant(QualType Ty, bool ExcludeCtor) {
   if (!Ty.isConstant(Context) && !Ty->isReferenceType())
     return false;
 
   if (Context.getLangOpts().CPlusPlus) {
     if (const CXXRecordDecl *Record
           = Context.getBaseElementType(Ty)->getAsCXXRecordDecl())
       return ExcludeCtor && !Record->hasMutableFields() &&
              Record->hasTrivialDestructor();
   }
 
   return true;
 }
 
 /// GetOrCreateLLVMGlobal - If the specified mangled name is not in the module,
 /// create and return an llvm GlobalVariable with the specified type and address
 /// space. If there is something in the module with the specified name, return
 /// it potentially bitcasted to the right type.
 ///
 /// If D is non-null, it specifies a decl that correspond to this.  This is used
 /// to set the attributes on the global when it is first created.
 ///
 /// If IsForDefinition is true, it is guaranteed that an actual global with
 /// type Ty will be returned, not conversion of a variable with the same
 /// mangled name but some other type.
 llvm::Constant *
 CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty,
                                      LangAS AddrSpace, const VarDecl *D,
                                      ForDefinition_t IsForDefinition) {
   // Lookup the entry, lazily creating it if necessary.
   llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
   unsigned TargetAS = getContext().getTargetAddressSpace(AddrSpace);
   if (Entry) {
     if (WeakRefReferences.erase(Entry)) {
       if (D && !D->hasAttr<WeakAttr>())
         Entry->setLinkage(llvm::Function::ExternalLinkage);
     }
 
     // Handle dropped DLL attributes.
     if (D && !D->hasAttr<DLLImportAttr>() && !D->hasAttr<DLLExportAttr>())
       Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
 
     if (LangOpts.OpenMP && !LangOpts.OpenMPSimd && D)
       getOpenMPRuntime().registerTargetGlobalVariable(D, Entry);
 
     if (Entry->getValueType() == Ty && Entry->getAddressSpace() == TargetAS)
       return Entry;
 
     // If there are two attempts to define the same mangled name, issue an
     // error.
     if (IsForDefinition && !Entry->isDeclaration()) {
       GlobalDecl OtherGD;
       const VarDecl *OtherD;
 
       // Check that D is not yet in DiagnosedConflictingDefinitions is required
       // to make sure that we issue an error only once.
       if (D && lookupRepresentativeDecl(MangledName, OtherGD) &&
           (D->getCanonicalDecl() != OtherGD.getCanonicalDecl().getDecl()) &&
           (OtherD = dyn_cast<VarDecl>(OtherGD.getDecl())) &&
           OtherD->hasInit() &&
           DiagnosedConflictingDefinitions.insert(D).second) {
         getDiags().Report(D->getLocation(), diag::err_duplicate_mangled_name)
             << MangledName;
         getDiags().Report(OtherGD.getDecl()->getLocation(),
                           diag::note_previous_definition);
       }
     }
 
     // Make sure the result is of the correct type.
     if (Entry->getType()->getAddressSpace() != TargetAS) {
       return llvm::ConstantExpr::getAddrSpaceCast(Entry,
                                                   Ty->getPointerTo(TargetAS));
     }
 
     // (If global is requested for a definition, we always need to create a new
     // global, not just return a bitcast.)
     if (!IsForDefinition)
       return llvm::ConstantExpr::getBitCast(Entry, Ty->getPointerTo(TargetAS));
   }
 
   auto DAddrSpace = GetGlobalVarAddressSpace(D);
 
   auto *GV = new llvm::GlobalVariable(
       getModule(), Ty, false, llvm::GlobalValue::ExternalLinkage, nullptr,
       MangledName, nullptr, llvm::GlobalVariable::NotThreadLocal,
       getContext().getTargetAddressSpace(DAddrSpace));
 
   // If we already created a global with the same mangled name (but different
   // type) before, take its name and remove it from its parent.
   if (Entry) {
     GV->takeName(Entry);
 
     if (!Entry->use_empty()) {
       llvm::Constant *NewPtrForOldDecl =
           llvm::ConstantExpr::getBitCast(GV, Entry->getType());
       Entry->replaceAllUsesWith(NewPtrForOldDecl);
     }
 
     Entry->eraseFromParent();
   }
 
   // This is the first use or definition of a mangled name.  If there is a
   // deferred decl with this name, remember that we need to emit it at the end
   // of the file.
   auto DDI = DeferredDecls.find(MangledName);
   if (DDI != DeferredDecls.end()) {
     // Move the potentially referenced deferred decl to the DeferredDeclsToEmit
     // list, and remove it from DeferredDecls (since we don't need it anymore).
     addDeferredDeclToEmit(DDI->second);
     DeferredDecls.erase(DDI);
   }
 
   // Handle things which are present even on external declarations.
   if (D) {
     if (LangOpts.OpenMP && !LangOpts.OpenMPSimd)
       getOpenMPRuntime().registerTargetGlobalVariable(D, GV);
 
     // FIXME: This code is overly simple and should be merged with other global
     // handling.
     GV->setConstant(isTypeConstant(D->getType(), false));
 
     GV->setAlignment(getContext().getDeclAlign(D).getAsAlign());
 
     setLinkageForGV(GV, D);
 
     if (D->getTLSKind()) {
       if (D->getTLSKind() == VarDecl::TLS_Dynamic)
         CXXThreadLocals.push_back(D);
       setTLSMode(GV, *D);
     }
 
     setGVProperties(GV, D);
 
     // If required by the ABI, treat declarations of static data members with
     // inline initializers as definitions.
     if (getContext().isMSStaticDataMemberInlineDefinition(D)) {
       EmitGlobalVarDefinition(D);
     }
 
     // Emit section information for extern variables.
     if (D->hasExternalStorage()) {
       if (const SectionAttr *SA = D->getAttr<SectionAttr>())
         GV->setSection(SA->getName());
     }
 
     // Handle XCore specific ABI requirements.
     if (getTriple().getArch() == llvm::Triple::xcore &&
         D->getLanguageLinkage() == CLanguageLinkage &&
         D->getType().isConstant(Context) &&
         isExternallyVisible(D->getLinkageAndVisibility().getLinkage()))
       GV->setSection(".cp.rodata");
 
     // Check if we a have a const declaration with an initializer, we may be
     // able to emit it as available_externally to expose it's value to the
     // optimizer.
     if (Context.getLangOpts().CPlusPlus && GV->hasExternalLinkage() &&
         D->getType().isConstQualified() && !GV->hasInitializer() &&
         !D->hasDefinition() && D->hasInit() && !D->hasAttr<DLLImportAttr>()) {
       const auto *Record =
           Context.getBaseElementType(D->getType())->getAsCXXRecordDecl();
       bool HasMutableFields = Record && Record->hasMutableFields();
       if (!HasMutableFields) {
         const VarDecl *InitDecl;
         const Expr *InitExpr = D->getAnyInitializer(InitDecl);
         if (InitExpr) {
           ConstantEmitter emitter(*this);
           llvm::Constant *Init = emitter.tryEmitForInitializer(*InitDecl);
           if (Init) {
             auto *InitType = Init->getType();
             if (GV->getValueType() != InitType) {
               // The type of the initializer does not match the definition.
               // This happens when an initializer has a different type from
               // the type of the global (because of padding at the end of a
               // structure for instance).
               GV->setName(StringRef());
               // Make a new global with the correct type, this is now guaranteed
               // to work.
               auto *NewGV = cast<llvm::GlobalVariable>(
                   GetAddrOfGlobalVar(D, InitType, IsForDefinition)
                       ->stripPointerCasts());
 
               // Erase the old global, since it is no longer used.
               GV->eraseFromParent();
               GV = NewGV;
             } else {
               GV->setInitializer(Init);
               GV->setConstant(true);
               GV->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage);
             }
             emitter.finalize(GV);
           }
         }
       }
     }
   }
 
   if (GV->isDeclaration()) {
     getTargetCodeGenInfo().setTargetAttributes(D, GV, *this);
     // External HIP managed variables needed to be recorded for transformation
     // in both device and host compilations.
     if (getLangOpts().CUDA && D && D->hasAttr<HIPManagedAttr>() &&
         D->hasExternalStorage())
       getCUDARuntime().handleVarRegistration(D, *GV);
   }
 
   LangAS ExpectedAS =
       D ? D->getType().getAddressSpace()
         : (LangOpts.OpenCL ? LangAS::opencl_global : LangAS::Default);
   assert(getContext().getTargetAddressSpace(ExpectedAS) == TargetAS);
   if (DAddrSpace != ExpectedAS) {
     return getTargetCodeGenInfo().performAddrSpaceCast(
         *this, GV, DAddrSpace, ExpectedAS, Ty->getPointerTo(TargetAS));
   }
 
   return GV;
 }
 
 llvm::Constant *
 CodeGenModule::GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition) {
   const Decl *D = GD.getDecl();
 
   if (isa<CXXConstructorDecl>(D) || isa<CXXDestructorDecl>(D))
     return getAddrOfCXXStructor(GD, /*FnInfo=*/nullptr, /*FnType=*/nullptr,
                                 /*DontDefer=*/false, IsForDefinition);
 
   if (isa<CXXMethodDecl>(D)) {
     auto FInfo =
         &getTypes().arrangeCXXMethodDeclaration(cast<CXXMethodDecl>(D));
     auto Ty = getTypes().GetFunctionType(*FInfo);
     return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false,
                              IsForDefinition);
   }
 
   if (isa<FunctionDecl>(D)) {
     const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
     llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
     return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false,
                              IsForDefinition);
   }
 
   return GetAddrOfGlobalVar(cast<VarDecl>(D), /*Ty=*/nullptr, IsForDefinition);
 }
 
 llvm::GlobalVariable *CodeGenModule::CreateOrReplaceCXXRuntimeVariable(
     StringRef Name, llvm::Type *Ty, llvm::GlobalValue::LinkageTypes Linkage,
     unsigned Alignment) {
   llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name);
   llvm::GlobalVariable *OldGV = nullptr;
 
   if (GV) {
     // Check if the variable has the right type.
     if (GV->getValueType() == Ty)
       return GV;
 
     // Because C++ name mangling, the only way we can end up with an already
     // existing global with the same name is if it has been declared extern "C".
     assert(GV->isDeclaration() && "Declaration has wrong type!");
     OldGV = GV;
   }
 
   // Create a new variable.
   GV = new llvm::GlobalVariable(getModule(), Ty, /*isConstant=*/true,
                                 Linkage, nullptr, Name);
 
   if (OldGV) {
     // Replace occurrences of the old variable if needed.
     GV->takeName(OldGV);
 
     if (!OldGV->use_empty()) {
       llvm::Constant *NewPtrForOldDecl =
       llvm::ConstantExpr::getBitCast(GV, OldGV->getType());
       OldGV->replaceAllUsesWith(NewPtrForOldDecl);
     }
 
     OldGV->eraseFromParent();
   }
 
   if (supportsCOMDAT() && GV->isWeakForLinker() &&
       !GV->hasAvailableExternallyLinkage())
     GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
 
   GV->setAlignment(llvm::MaybeAlign(Alignment));
 
   return GV;
 }
 
 /// GetAddrOfGlobalVar - Return the llvm::Constant for the address of the
 /// given global variable.  If Ty is non-null and if the global doesn't exist,
 /// then it will be created with the specified type instead of whatever the
 /// normal requested type would be. If IsForDefinition is true, it is guaranteed
 /// that an actual global with type Ty will be returned, not conversion of a
 /// variable with the same mangled name but some other type.
 llvm::Constant *CodeGenModule::GetAddrOfGlobalVar(const VarDecl *D,
                                                   llvm::Type *Ty,
                                            ForDefinition_t IsForDefinition) {
   assert(D->hasGlobalStorage() && "Not a global variable");
   QualType ASTTy = D->getType();
   if (!Ty)
     Ty = getTypes().ConvertTypeForMem(ASTTy);
 
   StringRef MangledName = getMangledName(D);
   return GetOrCreateLLVMGlobal(MangledName, Ty, ASTTy.getAddressSpace(), D,
                                IsForDefinition);
 }
 
 /// CreateRuntimeVariable - Create a new runtime global variable with the
 /// specified type and name.
 llvm::Constant *
 CodeGenModule::CreateRuntimeVariable(llvm::Type *Ty,
                                      StringRef Name) {
   LangAS AddrSpace = getContext().getLangOpts().OpenCL ? LangAS::opencl_global
                                                        : LangAS::Default;
   auto *Ret = GetOrCreateLLVMGlobal(Name, Ty, AddrSpace, nullptr);
   setDSOLocal(cast<llvm::GlobalValue>(Ret->stripPointerCasts()));
   return Ret;
 }
 
 void CodeGenModule::EmitTentativeDefinition(const VarDecl *D) {
   assert(!D->getInit() && "Cannot emit definite definitions here!");
 
   StringRef MangledName = getMangledName(D);
   llvm::GlobalValue *GV = GetGlobalValue(MangledName);
 
   // We already have a definition, not declaration, with the same mangled name.
   // Emitting of declaration is not required (and actually overwrites emitted
   // definition).
   if (GV && !GV->isDeclaration())
     return;
 
   // If we have not seen a reference to this variable yet, place it into the
   // deferred declarations table to be emitted if needed later.
   if (!MustBeEmitted(D) && !GV) {
       DeferredDecls[MangledName] = D;
       return;
   }
 
   // The tentative definition is the only definition.
   EmitGlobalVarDefinition(D);
 }
 
 void CodeGenModule::EmitExternalDeclaration(const VarDecl *D) {
   EmitExternalVarDeclaration(D);
 }
 
 CharUnits CodeGenModule::GetTargetTypeStoreSize(llvm::Type *Ty) const {
   return Context.toCharUnitsFromBits(
       getDataLayout().getTypeStoreSizeInBits(Ty));
 }
 
 LangAS CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D) {
   if (LangOpts.OpenCL) {
     LangAS AS = D ? D->getType().getAddressSpace() : LangAS::opencl_global;
     assert(AS == LangAS::opencl_global ||
            AS == LangAS::opencl_global_device ||
            AS == LangAS::opencl_global_host ||
            AS == LangAS::opencl_constant ||
            AS == LangAS::opencl_local ||
            AS >= LangAS::FirstTargetAddressSpace);
     return AS;
   }
 
   if (LangOpts.SYCLIsDevice &&
       (!D || D->getType().getAddressSpace() == LangAS::Default))
     return LangAS::sycl_global;
 
   if (LangOpts.CUDA && LangOpts.CUDAIsDevice) {
     if (D && D->hasAttr<CUDAConstantAttr>())
       return LangAS::cuda_constant;
     else if (D && D->hasAttr<CUDASharedAttr>())
       return LangAS::cuda_shared;
     else if (D && D->hasAttr<CUDADeviceAttr>())
       return LangAS::cuda_device;
     else if (D && D->getType().isConstQualified())
       return LangAS::cuda_constant;
     else
       return LangAS::cuda_device;
   }
 
   if (LangOpts.OpenMP) {
     LangAS AS;
     if (OpenMPRuntime->hasAllocateAttributeForGlobalVar(D, AS))
       return AS;
   }
   return getTargetCodeGenInfo().getGlobalVarAddressSpace(*this, D);
 }
 
 LangAS CodeGenModule::GetGlobalConstantAddressSpace() const {
   // OpenCL v1.2 s6.5.3: a string literal is in the constant address space.
   if (LangOpts.OpenCL)
     return LangAS::opencl_constant;
   if (LangOpts.SYCLIsDevice)
     return LangAS::sycl_global;
   if (LangOpts.HIP && LangOpts.CUDAIsDevice && getTriple().isSPIRV())
     // For HIPSPV map literals to cuda_device (maps to CrossWorkGroup in SPIR-V)
     // instead of default AS (maps to Generic in SPIR-V). Otherwise, we end up
     // with OpVariable instructions with Generic storage class which is not
     // allowed (SPIR-V V1.6 s3.42.8). Also, mapping literals to SPIR-V
     // UniformConstant storage class is not viable as pointers to it may not be
     // casted to Generic pointers which are used to model HIP's "flat" pointers.
     return LangAS::cuda_device;
   if (auto AS = getTarget().getConstantAddressSpace())
     return AS.getValue();
   return LangAS::Default;
 }
 
 // In address space agnostic languages, string literals are in default address
 // space in AST. However, certain targets (e.g. amdgcn) request them to be
 // emitted in constant address space in LLVM IR. To be consistent with other
 // parts of AST, string literal global variables in constant address space
 // need to be casted to default address space before being put into address
 // map and referenced by other part of CodeGen.
 // In OpenCL, string literals are in constant address space in AST, therefore
 // they should not be casted to default address space.
 static llvm::Constant *
 castStringLiteralToDefaultAddressSpace(CodeGenModule &CGM,
                                        llvm::GlobalVariable *GV) {
   llvm::Constant *Cast = GV;
   if (!CGM.getLangOpts().OpenCL) {
     auto AS = CGM.GetGlobalConstantAddressSpace();
     if (AS != LangAS::Default)
       Cast = CGM.getTargetCodeGenInfo().performAddrSpaceCast(
           CGM, GV, AS, LangAS::Default,
           GV->getValueType()->getPointerTo(
               CGM.getContext().getTargetAddressSpace(LangAS::Default)));
   }
   return Cast;
 }
 
 template<typename SomeDecl>
 void CodeGenModule::MaybeHandleStaticInExternC(const SomeDecl *D,
                                                llvm::GlobalValue *GV) {
   if (!getLangOpts().CPlusPlus)
     return;
 
   // Must have 'used' attribute, or else inline assembly can't rely on
   // the name existing.
   if (!D->template hasAttr<UsedAttr>())
     return;
 
   // Must have internal linkage and an ordinary name.
   if (!D->getIdentifier() || D->getFormalLinkage() != InternalLinkage)
     return;
 
   // Must be in an extern "C" context. Entities declared directly within
   // a record are not extern "C" even if the record is in such a context.
   const SomeDecl *First = D->getFirstDecl();
   if (First->getDeclContext()->isRecord() || !First->isInExternCContext())
     return;
 
   // OK, this is an internal linkage entity inside an extern "C" linkage
   // specification. Make a note of that so we can give it the "expected"
   // mangled name if nothing else is using that name.
   std::pair<StaticExternCMap::iterator, bool> R =
       StaticExternCValues.insert(std::make_pair(D->getIdentifier(), GV));
 
   // If we have multiple internal linkage entities with the same name
   // in extern "C" regions, none of them gets that name.
   if (!R.second)
     R.first->second = nullptr;
 }
 
 static bool shouldBeInCOMDAT(CodeGenModule &CGM, const Decl &D) {
   if (!CGM.supportsCOMDAT())
     return false;
 
   if (D.hasAttr<SelectAnyAttr>())
     return true;
 
   GVALinkage Linkage;
   if (auto *VD = dyn_cast<VarDecl>(&D))
     Linkage = CGM.getContext().GetGVALinkageForVariable(VD);
   else
     Linkage = CGM.getContext().GetGVALinkageForFunction(cast<FunctionDecl>(&D));
 
   switch (Linkage) {
   case GVA_Internal:
   case GVA_AvailableExternally:
   case GVA_StrongExternal:
     return false;
   case GVA_DiscardableODR:
   case GVA_StrongODR:
     return true;
   }
   llvm_unreachable("No such linkage");
 }
 
 void CodeGenModule::maybeSetTrivialComdat(const Decl &D,
                                           llvm::GlobalObject &GO) {
   if (!shouldBeInCOMDAT(*this, D))
     return;
   GO.setComdat(TheModule.getOrInsertComdat(GO.getName()));
 }
 
 /// Pass IsTentative as true if you want to create a tentative definition.
 void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
                                             bool IsTentative) {
   // OpenCL global variables of sampler type are translated to function calls,
   // therefore no need to be translated.
   QualType ASTTy = D->getType();
   if (getLangOpts().OpenCL && ASTTy->isSamplerT())
     return;
 
   // If this is OpenMP device, check if it is legal to emit this global
   // normally.
   if (LangOpts.OpenMPIsDevice && OpenMPRuntime &&
       OpenMPRuntime->emitTargetGlobalVariable(D))
     return;
 
   llvm::TrackingVH<llvm::Constant> Init;
   bool NeedsGlobalCtor = false;
   bool NeedsGlobalDtor =
       D->needsDestruction(getContext()) == QualType::DK_cxx_destructor;
 
   const VarDecl *InitDecl;
   const Expr *InitExpr = D->getAnyInitializer(InitDecl);
 
   Optional<ConstantEmitter> emitter;
 
   // CUDA E.2.4.1 "__shared__ variables cannot have an initialization
   // as part of their declaration."  Sema has already checked for
   // error cases, so we just need to set Init to UndefValue.
   bool IsCUDASharedVar =
       getLangOpts().CUDAIsDevice && D->hasAttr<CUDASharedAttr>();
   // Shadows of initialized device-side global variables are also left
   // undefined.
   // Managed Variables should be initialized on both host side and device side.
   bool IsCUDAShadowVar =
       !getLangOpts().CUDAIsDevice && !D->hasAttr<HIPManagedAttr>() &&
       (D->hasAttr<CUDAConstantAttr>() || D->hasAttr<CUDADeviceAttr>() ||
        D->hasAttr<CUDASharedAttr>());
   bool IsCUDADeviceShadowVar =
       getLangOpts().CUDAIsDevice && !D->hasAttr<HIPManagedAttr>() &&
       (D->getType()->isCUDADeviceBuiltinSurfaceType() ||
        D->getType()->isCUDADeviceBuiltinTextureType());
   if (getLangOpts().CUDA &&
       (IsCUDASharedVar || IsCUDAShadowVar || IsCUDADeviceShadowVar))
     Init = llvm::UndefValue::get(getTypes().ConvertTypeForMem(ASTTy));
   else if (D->hasAttr<LoaderUninitializedAttr>())
     Init = llvm::UndefValue::get(getTypes().ConvertTypeForMem(ASTTy));
   else if (!InitExpr) {
     // This is a tentative definition; tentative definitions are
     // implicitly initialized with { 0 }.
     //
     // Note that tentative definitions are only emitted at the end of
     // a translation unit, so they should never have incomplete
     // type. In addition, EmitTentativeDefinition makes sure that we
     // never attempt to emit a tentative definition if a real one
     // exists. A use may still exists, however, so we still may need
     // to do a RAUW.
     assert(!ASTTy->isIncompleteType() && "Unexpected incomplete type");
     Init = EmitNullConstant(D->getType());
   } else {
     initializedGlobalDecl = GlobalDecl(D);
     emitter.emplace(*this);
     llvm::Constant *Initializer = emitter->tryEmitForInitializer(*InitDecl);
     if (!Initializer) {
       QualType T = InitExpr->getType();
       if (D->getType()->isReferenceType())
         T = D->getType();
 
       if (getLangOpts().CPlusPlus) {
         Init = EmitNullConstant(T);
         NeedsGlobalCtor = true;
       } else {
         ErrorUnsupported(D, "static initializer");
         Init = llvm::UndefValue::get(getTypes().ConvertType(T));
       }
     } else {
       Init = Initializer;
       // We don't need an initializer, so remove the entry for the delayed
       // initializer position (just in case this entry was delayed) if we
       // also don't need to register a destructor.
       if (getLangOpts().CPlusPlus && !NeedsGlobalDtor)
         DelayedCXXInitPosition.erase(D);
     }
   }
 
   llvm::Type* InitType = Init->getType();
   llvm::Constant *Entry =
       GetAddrOfGlobalVar(D, InitType, ForDefinition_t(!IsTentative));
 
   // Strip off pointer casts if we got them.
   Entry = Entry->stripPointerCasts();
 
   // Entry is now either a Function or GlobalVariable.
   auto *GV = dyn_cast<llvm::GlobalVariable>(Entry);
 
   // We have a definition after a declaration with the wrong type.
   // We must make a new GlobalVariable* and update everything that used OldGV
   // (a declaration or tentative definition) with the new GlobalVariable*
   // (which will be a definition).
   //
   // This happens if there is a prototype for a global (e.g.
   // "extern int x[];") and then a definition of a different type (e.g.
   // "int x[10];"). This also happens when an initializer has a different type
   // from the type of the global (this happens with unions).
   if (!GV || GV->getValueType() != InitType ||
       GV->getType()->getAddressSpace() !=
           getContext().getTargetAddressSpace(GetGlobalVarAddressSpace(D))) {
 
     // Move the old entry aside so that we'll create a new one.
     Entry->setName(StringRef());
 
     // Make a new global with the correct type, this is now guaranteed to work.
     GV = cast<llvm::GlobalVariable>(
         GetAddrOfGlobalVar(D, InitType, ForDefinition_t(!IsTentative))
             ->stripPointerCasts());
 
     // Replace all uses of the old global with the new global
     llvm::Constant *NewPtrForOldDecl =
         llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV,
                                                              Entry->getType());
     Entry->replaceAllUsesWith(NewPtrForOldDecl);
 
     // Erase the old global, since it is no longer used.
     cast<llvm::GlobalValue>(Entry)->eraseFromParent();
   }
 
   MaybeHandleStaticInExternC(D, GV);
 
   if (D->hasAttr<AnnotateAttr>())
     AddGlobalAnnotations(D, GV);
 
   // Set the llvm linkage type as appropriate.
   llvm::GlobalValue::LinkageTypes Linkage =
       getLLVMLinkageVarDefinition(D, GV->isConstant());
 
   // CUDA B.2.1 "The __device__ qualifier declares a variable that resides on
   // the device. [...]"
   // CUDA B.2.2 "The __constant__ qualifier, optionally used together with
   // __device__, declares a variable that: [...]
   // Is accessible from all the threads within the grid and from the host
   // through the runtime library (cudaGetSymbolAddress() / cudaGetSymbolSize()
   // / cudaMemcpyToSymbol() / cudaMemcpyFromSymbol())."
   if (GV && LangOpts.CUDA) {
     if (LangOpts.CUDAIsDevice) {
       if (Linkage != llvm::GlobalValue::InternalLinkage &&
           (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() ||
            D->getType()->isCUDADeviceBuiltinSurfaceType() ||
            D->getType()->isCUDADeviceBuiltinTextureType()))
         GV->setExternallyInitialized(true);
     } else {
       getCUDARuntime().internalizeDeviceSideVar(D, Linkage);
     }
     getCUDARuntime().handleVarRegistration(D, *GV);
   }
 
   GV->setInitializer(Init);
   if (emitter)
     emitter->finalize(GV);
 
   // If it is safe to mark the global 'constant', do so now.
   GV->setConstant(!NeedsGlobalCtor && !NeedsGlobalDtor &&
                   isTypeConstant(D->getType(), true));
 
   // If it is in a read-only section, mark it 'constant'.
   if (const SectionAttr *SA = D->getAttr<SectionAttr>()) {
     const ASTContext::SectionInfo &SI = Context.SectionInfos[SA->getName()];
     if ((SI.SectionFlags & ASTContext::PSF_Write) == 0)
       GV->setConstant(true);
   }
 
   GV->setAlignment(getContext().getDeclAlign(D).getAsAlign());
 
   // On Darwin, unlike other Itanium C++ ABI platforms, the thread-wrapper
   // function is only defined alongside the variable, not also alongside
   // callers. Normally, all accesses to a thread_local go through the
   // thread-wrapper in order to ensure initialization has occurred, underlying
   // variable will never be used other than the thread-wrapper, so it can be
   // converted to internal linkage.
   //
   // However, if the variable has the 'constinit' attribute, it _can_ be
   // referenced directly, without calling the thread-wrapper, so the linkage
   // must not be changed.
   //
   // Additionally, if the variable isn't plain external linkage, e.g. if it's
   // weak or linkonce, the de-duplication semantics are important to preserve,
   // so we don't change the linkage.
   if (D->getTLSKind() == VarDecl::TLS_Dynamic &&
       Linkage == llvm::GlobalValue::ExternalLinkage &&
       Context.getTargetInfo().getTriple().isOSDarwin() &&
       !D->hasAttr<ConstInitAttr>())
     Linkage = llvm::GlobalValue::InternalLinkage;
 
   GV->setLinkage(Linkage);
   if (D->hasAttr<DLLImportAttr>())
     GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass);
   else if (D->hasAttr<DLLExportAttr>())
     GV->setDLLStorageClass(llvm::GlobalVariable::DLLExportStorageClass);
   else
     GV->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass);
 
   if (Linkage == llvm::GlobalVariable::CommonLinkage) {
     // common vars aren't constant even if declared const.
     GV->setConstant(false);
     // Tentative definition of global variables may be initialized with
     // non-zero null pointers. In this case they should have weak linkage
     // since common linkage must have zero initializer and must not have
     // explicit section therefore cannot have non-zero initial value.
     if (!GV->getInitializer()->isNullValue())
       GV->setLinkage(llvm::GlobalVariable::WeakAnyLinkage);
   }
 
   setNonAliasAttributes(D, GV);
 
   if (D->getTLSKind() && !GV->isThreadLocal()) {
     if (D->getTLSKind() == VarDecl::TLS_Dynamic)
       CXXThreadLocals.push_back(D);
     setTLSMode(GV, *D);
   }
 
   maybeSetTrivialComdat(*D, *GV);
 
   // Emit the initializer function if necessary.
   if (NeedsGlobalCtor || NeedsGlobalDtor)
     EmitCXXGlobalVarDeclInitFunc(D, GV, NeedsGlobalCtor);
 
   SanitizerMD->reportGlobalToASan(GV, *D, NeedsGlobalCtor);
 
   // Emit global variable debug information.
   if (CGDebugInfo *DI = getModuleDebugInfo())
     if (getCodeGenOpts().hasReducedDebugInfo())
       DI->EmitGlobalVariable(GV, D);
 }
 
 void CodeGenModule::EmitExternalVarDeclaration(const VarDecl *D) {
   if (CGDebugInfo *DI = getModuleDebugInfo())
     if (getCodeGenOpts().hasReducedDebugInfo()) {
       QualType ASTTy = D->getType();
       llvm::Type *Ty = getTypes().ConvertTypeForMem(D->getType());
       llvm::Constant *GV =
           GetOrCreateLLVMGlobal(D->getName(), Ty, ASTTy.getAddressSpace(), D);
       DI->EmitExternalVariable(
           cast<llvm::GlobalVariable>(GV->stripPointerCasts()), D);
     }
 }
 
 static bool isVarDeclStrongDefinition(const ASTContext &Context,
                                       CodeGenModule &CGM, const VarDecl *D,
                                       bool NoCommon) {
   // Don't give variables common linkage if -fno-common was specified unless it
   // was overridden by a NoCommon attribute.
   if ((NoCommon || D->hasAttr<NoCommonAttr>()) && !D->hasAttr<CommonAttr>())
     return true;
 
   // C11 6.9.2/2:
   //   A declaration of an identifier for an object that has file scope without
   //   an initializer, and without a storage-class specifier or with the
   //   storage-class specifier static, constitutes a tentative definition.
   if (D->getInit() || D->hasExternalStorage())
     return true;
 
   // A variable cannot be both common and exist in a section.
   if (D->hasAttr<SectionAttr>())
     return true;
 
   // A variable cannot be both common and exist in a section.
   // We don't try to determine which is the right section in the front-end.
   // If no specialized section name is applicable, it will resort to default.
   if (D->hasAttr<PragmaClangBSSSectionAttr>() ||
       D->hasAttr<PragmaClangDataSectionAttr>() ||
       D->hasAttr<PragmaClangRelroSectionAttr>() ||
       D->hasAttr<PragmaClangRodataSectionAttr>())
     return true;
 
   // Thread local vars aren't considered common linkage.
   if (D->getTLSKind())
     return true;
 
   // Tentative definitions marked with WeakImportAttr are true definitions.
   if (D->hasAttr<WeakImportAttr>())
     return true;
 
   // A variable cannot be both common and exist in a comdat.
   if (shouldBeInCOMDAT(CGM, *D))
     return true;
 
   // Declarations with a required alignment do not have common linkage in MSVC
   // mode.
   if (Context.getTargetInfo().getCXXABI().isMicrosoft()) {
     if (D->hasAttr<AlignedAttr>())
       return true;
     QualType VarType = D->getType();
     if (Context.isAlignmentRequired(VarType))
       return true;
 
     if (const auto *RT = VarType->getAs<RecordType>()) {
       const RecordDecl *RD = RT->getDecl();
       for (const FieldDecl *FD : RD->fields()) {
         if (FD->isBitField())
           continue;
         if (FD->hasAttr<AlignedAttr>())
           return true;
         if (Context.isAlignmentRequired(FD->getType()))
           return true;
       }
     }
   }
 
   // Microsoft's link.exe doesn't support alignments greater than 32 bytes for
   // common symbols, so symbols with greater alignment requirements cannot be
   // common.
   // Other COFF linkers (ld.bfd and LLD) support arbitrary power-of-two
   // alignments for common symbols via the aligncomm directive, so this
   // restriction only applies to MSVC environments.
   if (Context.getTargetInfo().getTriple().isKnownWindowsMSVCEnvironment() &&
       Context.getTypeAlignIfKnown(D->getType()) >
           Context.toBits(CharUnits::fromQuantity(32)))
     return true;
 
   return false;
 }
 
 llvm::GlobalValue::LinkageTypes CodeGenModule::getLLVMLinkageForDeclarator(
     const DeclaratorDecl *D, GVALinkage Linkage, bool IsConstantVariable) {
   if (Linkage == GVA_Internal)
     return llvm::Function::InternalLinkage;
 
   if (D->hasAttr<WeakAttr>()) {
     if (IsConstantVariable)
       return llvm::GlobalVariable::WeakODRLinkage;
     else
       return llvm::GlobalVariable::WeakAnyLinkage;
   }
 
   if (const auto *FD = D->getAsFunction())
     if (FD->isMultiVersion() && Linkage == GVA_AvailableExternally)
       return llvm::GlobalVariable::LinkOnceAnyLinkage;
 
   // We are guaranteed to have a strong definition somewhere else,
   // so we can use available_externally linkage.
   if (Linkage == GVA_AvailableExternally)
     return llvm::GlobalValue::AvailableExternallyLinkage;
 
   // Note that Apple's kernel linker doesn't support symbol
   // coalescing, so we need to avoid linkonce and weak linkages there.
   // Normally, this means we just map to internal, but for explicit
   // instantiations we'll map to external.
 
   // In C++, the compiler has to emit a definition in every translation unit
   // that references the function.  We should use linkonce_odr because
   // a) if all references in this translation unit are optimized away, we
   // don't need to codegen it.  b) if the function persists, it needs to be
   // merged with other definitions. c) C++ has the ODR, so we know the
   // definition is dependable.
   if (Linkage == GVA_DiscardableODR)
     return !Context.getLangOpts().AppleKext ? llvm::Function::LinkOnceODRLinkage
                                             : llvm::Function::InternalLinkage;
 
   // An explicit instantiation of a template has weak linkage, since
   // explicit instantiations can occur in multiple translation units
   // and must all be equivalent. However, we are not allowed to
   // throw away these explicit instantiations.
   //
   // CUDA/HIP: For -fno-gpu-rdc case, device code is limited to one TU,
   // so say that CUDA templates are either external (for kernels) or internal.
   // This lets llvm perform aggressive inter-procedural optimizations. For
   // -fgpu-rdc case, device function calls across multiple TU's are allowed,
   // therefore we need to follow the normal linkage paradigm.
   if (Linkage == GVA_StrongODR) {
     if (getLangOpts().AppleKext)
       return llvm::Function::ExternalLinkage;
     if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice &&
         !getLangOpts().GPURelocatableDeviceCode)
       return D->hasAttr<CUDAGlobalAttr>() ? llvm::Function::ExternalLinkage
                                           : llvm::Function::InternalLinkage;
     return llvm::Function::WeakODRLinkage;
   }
 
   // C++ doesn't have tentative definitions and thus cannot have common
   // linkage.
   if (!getLangOpts().CPlusPlus && isa<VarDecl>(D) &&
       !isVarDeclStrongDefinition(Context, *this, cast<VarDecl>(D),
                                  CodeGenOpts.NoCommon))
     return llvm::GlobalVariable::CommonLinkage;
 
   // selectany symbols are externally visible, so use weak instead of
   // linkonce.  MSVC optimizes away references to const selectany globals, so
   // all definitions should be the same and ODR linkage should be used.
   // http://msdn.microsoft.com/en-us/library/5tkz6s71.aspx
   if (D->hasAttr<SelectAnyAttr>())
     return llvm::GlobalVariable::WeakODRLinkage;
 
   // Otherwise, we have strong external linkage.
   assert(Linkage == GVA_StrongExternal);
   return llvm::GlobalVariable::ExternalLinkage;
 }
 
 llvm::GlobalValue::LinkageTypes CodeGenModule::getLLVMLinkageVarDefinition(
     const VarDecl *VD, bool IsConstant) {
   GVALinkage Linkage = getContext().GetGVALinkageForVariable(VD);
   return getLLVMLinkageForDeclarator(VD, Linkage, IsConstant);
 }
 
 /// Replace the uses of a function that was declared with a non-proto type.
 /// We want to silently drop extra arguments from call sites
 static void replaceUsesOfNonProtoConstant(llvm::Constant *old,
                                           llvm::Function *newFn) {
   // Fast path.
   if (old->use_empty()) return;
 
   llvm::Type *newRetTy = newFn->getReturnType();
   SmallVector<llvm::Value*, 4> newArgs;
 
   for (llvm::Value::use_iterator ui = old->use_begin(), ue = old->use_end();
          ui != ue; ) {
     llvm::Value::use_iterator use = ui++; // Increment before the use is erased.
     llvm::User *user = use->getUser();
 
     // Recognize and replace uses of bitcasts.  Most calls to
     // unprototyped functions will use bitcasts.
     if (auto *bitcast = dyn_cast<llvm::ConstantExpr>(user)) {
       if (bitcast->getOpcode() == llvm::Instruction::BitCast)
         replaceUsesOfNonProtoConstant(bitcast, newFn);
       continue;
     }
 
     // Recognize calls to the function.
     llvm::CallBase *callSite = dyn_cast<llvm::CallBase>(user);
     if (!callSite) continue;
     if (!callSite->isCallee(&*use))
       continue;
 
     // If the return types don't match exactly, then we can't
     // transform this call unless it's dead.
     if (callSite->getType() != newRetTy && !callSite->use_empty())
       continue;
 
     // Get the call site's attribute list.
     SmallVector<llvm::AttributeSet, 8> newArgAttrs;
     llvm::AttributeList oldAttrs = callSite->getAttributes();
 
     // If the function was passed too few arguments, don't transform.
     unsigned newNumArgs = newFn->arg_size();
     if (callSite->arg_size() < newNumArgs)
       continue;
 
     // If extra arguments were passed, we silently drop them.
     // If any of the types mismatch, we don't transform.
     unsigned argNo = 0;
     bool dontTransform = false;
     for (llvm::Argument &A : newFn->args()) {
       if (callSite->getArgOperand(argNo)->getType() != A.getType()) {
         dontTransform = true;
         break;
       }
 
       // Add any parameter attributes.
       newArgAttrs.push_back(oldAttrs.getParamAttrs(argNo));
       argNo++;
     }
     if (dontTransform)
       continue;
 
     // Okay, we can transform this.  Create the new call instruction and copy
     // over the required information.
     newArgs.append(callSite->arg_begin(), callSite->arg_begin() + argNo);
 
     // Copy over any operand bundles.
     SmallVector<llvm::OperandBundleDef, 1> newBundles;
     callSite->getOperandBundlesAsDefs(newBundles);
 
     llvm::CallBase *newCall;
     if (isa<llvm::CallInst>(callSite)) {
       newCall =
           llvm::CallInst::Create(newFn, newArgs, newBundles, "", callSite);
     } else {
       auto *oldInvoke = cast<llvm::InvokeInst>(callSite);
       newCall = llvm::InvokeInst::Create(newFn, oldInvoke->getNormalDest(),
                                          oldInvoke->getUnwindDest(), newArgs,
                                          newBundles, "", callSite);
     }
     newArgs.clear(); // for the next iteration
 
     if (!newCall->getType()->isVoidTy())
       newCall->takeName(callSite);
     newCall->setAttributes(
         llvm::AttributeList::get(newFn->getContext(), oldAttrs.getFnAttrs(),
                                  oldAttrs.getRetAttrs(), newArgAttrs));
     newCall->setCallingConv(callSite->getCallingConv());
 
     // Finally, remove the old call, replacing any uses with the new one.
     if (!callSite->use_empty())
       callSite->replaceAllUsesWith(newCall);
 
     // Copy debug location attached to CI.
     if (callSite->getDebugLoc())
       newCall->setDebugLoc(callSite->getDebugLoc());
 
     callSite->eraseFromParent();
   }
 }
 
 /// ReplaceUsesOfNonProtoTypeWithRealFunction - This function is called when we
 /// implement a function with no prototype, e.g. "int foo() {}".  If there are
 /// existing call uses of the old function in the module, this adjusts them to
 /// call the new function directly.
 ///
 /// This is not just a cleanup: the always_inline pass requires direct calls to
 /// functions to be able to inline them.  If there is a bitcast in the way, it
 /// won't inline them.  Instcombine normally deletes these calls, but it isn't
 /// run at -O0.
 static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
                                                       llvm::Function *NewFn) {
   // If we're redefining a global as a function, don't transform it.
   if (!isa<llvm::Function>(Old)) return;
 
   replaceUsesOfNonProtoConstant(Old, NewFn);
 }
 
 void CodeGenModule::HandleCXXStaticMemberVarInstantiation(VarDecl *VD) {
   auto DK = VD->isThisDeclarationADefinition();
   if (DK == VarDecl::Definition && VD->hasAttr<DLLImportAttr>())
     return;
 
   TemplateSpecializationKind TSK = VD->getTemplateSpecializationKind();
   // If we have a definition, this might be a deferred decl. If the
   // instantiation is explicit, make sure we emit it at the end.
   if (VD->getDefinition() && TSK == TSK_ExplicitInstantiationDefinition)
     GetAddrOfGlobalVar(VD);
 
   EmitTopLevelDecl(VD);
 }
 
 void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD,
                                                  llvm::GlobalValue *GV) {
   const auto *D = cast<FunctionDecl>(GD.getDecl());
 
   // Compute the function info and LLVM type.
   const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
   llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
 
   // Get or create the prototype for the function.
   if (!GV || (GV->getValueType() != Ty))
     GV = cast<llvm::GlobalValue>(GetAddrOfFunction(GD, Ty, /*ForVTable=*/false,
                                                    /*DontDefer=*/true,
                                                    ForDefinition));
 
   // Already emitted.
   if (!GV->isDeclaration())
     return;
 
   // We need to set linkage and visibility on the function before
   // generating code for it because various parts of IR generation
   // want to propagate this information down (e.g. to local static
   // declarations).
   auto *Fn = cast<llvm::Function>(GV);
   setFunctionLinkage(GD, Fn);
 
   // FIXME: this is redundant with part of setFunctionDefinitionAttributes
   setGVProperties(Fn, GD);
 
   MaybeHandleStaticInExternC(D, Fn);
 
   maybeSetTrivialComdat(*D, *Fn);
 
   // Set CodeGen attributes that represent floating point environment.
   setLLVMFunctionFEnvAttributes(D, Fn);
 
   CodeGenFunction(*this).GenerateCode(GD, Fn, FI);
 
   setNonAliasAttributes(GD, Fn);
   SetLLVMFunctionAttributesForDefinition(D, Fn);
 
   if (const ConstructorAttr *CA = D->getAttr<ConstructorAttr>())
     AddGlobalCtor(Fn, CA->getPriority());
   if (const DestructorAttr *DA = D->getAttr<DestructorAttr>())
     AddGlobalDtor(Fn, DA->getPriority(), true);
   if (D->hasAttr<AnnotateAttr>())
     AddGlobalAnnotations(D, Fn);
 }
 
 void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) {
   const auto *D = cast<ValueDecl>(GD.getDecl());
   const AliasAttr *AA = D->getAttr<AliasAttr>();
   assert(AA && "Not an alias?");
 
   StringRef MangledName = getMangledName(GD);
 
   if (AA->getAliasee() == MangledName) {
     Diags.Report(AA->getLocation(), diag::err_cyclic_alias) << 0;
     return;
   }
 
   // If there is a definition in the module, then it wins over the alias.
   // This is dubious, but allow it to be safe.  Just ignore the alias.
   llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
   if (Entry && !Entry->isDeclaration())
     return;
 
   Aliases.push_back(GD);
 
   llvm::Type *DeclTy = getTypes().ConvertTypeForMem(D->getType());
 
   // Create a reference to the named value.  This ensures that it is emitted
   // if a deferred decl.
   llvm::Constant *Aliasee;
   llvm::GlobalValue::LinkageTypes LT;
   if (isa<llvm::FunctionType>(DeclTy)) {
     Aliasee = GetOrCreateLLVMFunction(AA->getAliasee(), DeclTy, GD,
                                       /*ForVTable=*/false);
     LT = getFunctionLinkage(GD);
   } else {
     Aliasee = GetOrCreateLLVMGlobal(AA->getAliasee(), DeclTy, LangAS::Default,
                                     /*D=*/nullptr);
     if (const auto *VD = dyn_cast<VarDecl>(GD.getDecl()))
       LT = getLLVMLinkageVarDefinition(VD, D->getType().isConstQualified());
     else
       LT = getFunctionLinkage(GD);
   }
 
   // Create the new alias itself, but don't set a name yet.
   unsigned AS = Aliasee->getType()->getPointerAddressSpace();
   auto *GA =
       llvm::GlobalAlias::create(DeclTy, AS, LT, "", Aliasee, &getModule());
 
   if (Entry) {
     if (GA->getAliasee() == Entry) {
       Diags.Report(AA->getLocation(), diag::err_cyclic_alias) << 0;
       return;
     }
 
     assert(Entry->isDeclaration());
 
     // If there is a declaration in the module, then we had an extern followed
     // by the alias, as in:
     //   extern int test6();
     //   ...
     //   int test6() __attribute__((alias("test7")));
     //
     // Remove it and replace uses of it with the alias.
     GA->takeName(Entry);
 
     Entry->replaceAllUsesWith(llvm::ConstantExpr::getBitCast(GA,
                                                           Entry->getType()));
     Entry->eraseFromParent();
   } else {
     GA->setName(MangledName);
   }
 
   // Set attributes which are particular to an alias; this is a
   // specialization of the attributes which may be set on a global
   // variable/function.
   if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() ||
       D->isWeakImported()) {
     GA->setLinkage(llvm::Function::WeakAnyLinkage);
   }
 
   if (const auto *VD = dyn_cast<VarDecl>(D))
     if (VD->getTLSKind())
       setTLSMode(GA, *VD);
 
   SetCommonAttributes(GD, GA);
 }
 
 void CodeGenModule::emitIFuncDefinition(GlobalDecl GD) {
   const auto *D = cast<ValueDecl>(GD.getDecl());
   const IFuncAttr *IFA = D->getAttr<IFuncAttr>();
   assert(IFA && "Not an ifunc?");
 
   StringRef MangledName = getMangledName(GD);
 
   if (IFA->getResolver() == MangledName) {
     Diags.Report(IFA->getLocation(), diag::err_cyclic_alias) << 1;
     return;
   }
 
   // Report an error if some definition overrides ifunc.
   llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
   if (Entry && !Entry->isDeclaration()) {
     GlobalDecl OtherGD;
     if (lookupRepresentativeDecl(MangledName, OtherGD) &&
         DiagnosedConflictingDefinitions.insert(GD).second) {
       Diags.Report(D->getLocation(), diag::err_duplicate_mangled_name)
           << MangledName;
       Diags.Report(OtherGD.getDecl()->getLocation(),
                    diag::note_previous_definition);
     }
     return;
   }
 
   Aliases.push_back(GD);
 
   llvm::Type *DeclTy = getTypes().ConvertTypeForMem(D->getType());
   llvm::Type *ResolverTy = llvm::GlobalIFunc::getResolverFunctionType(DeclTy);
   llvm::Constant *Resolver =
       GetOrCreateLLVMFunction(IFA->getResolver(), ResolverTy, {},
                               /*ForVTable=*/false);
   llvm::GlobalIFunc *GIF =
       llvm::GlobalIFunc::create(DeclTy, 0, llvm::Function::ExternalLinkage,
                                 "", Resolver, &getModule());
   if (Entry) {
     if (GIF->getResolver() == Entry) {
       Diags.Report(IFA->getLocation(), diag::err_cyclic_alias) << 1;
       return;
     }
     assert(Entry->isDeclaration());
 
     // If there is a declaration in the module, then we had an extern followed
     // by the ifunc, as in:
     //   extern int test();
     //   ...
     //   int test() __attribute__((ifunc("resolver")));
     //
     // Remove it and replace uses of it with the ifunc.
     GIF->takeName(Entry);
 
     Entry->replaceAllUsesWith(llvm::ConstantExpr::getBitCast(GIF,
                                                           Entry->getType()));
     Entry->eraseFromParent();
   } else
     GIF->setName(MangledName);
 
   SetCommonAttributes(GD, GIF);
 }
 
 llvm::Function *CodeGenModule::getIntrinsic(unsigned IID,
                                             ArrayRef<llvm::Type*> Tys) {
   return llvm::Intrinsic::getDeclaration(&getModule(), (llvm::Intrinsic::ID)IID,
                                          Tys);
 }
 
 static llvm::StringMapEntry<llvm::GlobalVariable *> &
 GetConstantCFStringEntry(llvm::StringMap<llvm::GlobalVariable *> &Map,
                          const StringLiteral *Literal, bool TargetIsLSB,
                          bool &IsUTF16, unsigned &StringLength) {
   StringRef String = Literal->getString();
   unsigned NumBytes = String.size();
 
   // Check for simple case.
   if (!Literal->containsNonAsciiOrNull()) {
     StringLength = NumBytes;
     return *Map.insert(std::make_pair(String, nullptr)).first;
   }
 
   // Otherwise, convert the UTF8 literals into a string of shorts.
   IsUTF16 = true;
 
   SmallVector<llvm::UTF16, 128> ToBuf(NumBytes + 1); // +1 for ending nulls.
   const llvm::UTF8 *FromPtr = (const llvm::UTF8 *)String.data();
   llvm::UTF16 *ToPtr = &ToBuf[0];
 
   (void)llvm::ConvertUTF8toUTF16(&FromPtr, FromPtr + NumBytes, &ToPtr,
                                  ToPtr + NumBytes, llvm::strictConversion);
 
   // ConvertUTF8toUTF16 returns the length in ToPtr.
   StringLength = ToPtr - &ToBuf[0];
 
   // Add an explicit null.
   *ToPtr = 0;
   return *Map.insert(std::make_pair(
                          StringRef(reinterpret_cast<const char *>(ToBuf.data()),
                                    (StringLength + 1) * 2),
                          nullptr)).first;
 }
 
 ConstantAddress
 CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
   unsigned StringLength = 0;
   bool isUTF16 = false;
   llvm::StringMapEntry<llvm::GlobalVariable *> &Entry =
       GetConstantCFStringEntry(CFConstantStringMap, Literal,
                                getDataLayout().isLittleEndian(), isUTF16,
                                StringLength);
 
   if (auto *C = Entry.second)
     return ConstantAddress(
         C, C->getValueType(), CharUnits::fromQuantity(C->getAlignment()));
 
   llvm::Constant *Zero = llvm::Constant::getNullValue(Int32Ty);
   llvm::Constant *Zeros[] = { Zero, Zero };
 
   const ASTContext &Context = getContext();
   const llvm::Triple &Triple = getTriple();
 
   const auto CFRuntime = getLangOpts().CFRuntime;
   const bool IsSwiftABI =
       static_cast<unsigned>(CFRuntime) >=
       static_cast<unsigned>(LangOptions::CoreFoundationABI::Swift);
   const bool IsSwift4_1 = CFRuntime == LangOptions::CoreFoundationABI::Swift4_1;
 
   // If we don't already have it, get __CFConstantStringClassReference.
   if (!CFConstantStringClassRef) {
     const char *CFConstantStringClassName = "__CFConstantStringClassReference";
     llvm::Type *Ty = getTypes().ConvertType(getContext().IntTy);
     Ty = llvm::ArrayType::get(Ty, 0);
 
     switch (CFRuntime) {
     default: break;
     case LangOptions::CoreFoundationABI::Swift: LLVM_FALLTHROUGH;
     case LangOptions::CoreFoundationABI::Swift5_0:
       CFConstantStringClassName =
           Triple.isOSDarwin() ? "$s15SwiftFoundation19_NSCFConstantStringCN"
                               : "$s10Foundation19_NSCFConstantStringCN";
       Ty = IntPtrTy;
       break;
     case LangOptions::CoreFoundationABI::Swift4_2:
       CFConstantStringClassName =
           Triple.isOSDarwin() ? "$S15SwiftFoundation19_NSCFConstantStringCN"
                               : "$S10Foundation19_NSCFConstantStringCN";
       Ty = IntPtrTy;
       break;
     case LangOptions::CoreFoundationABI::Swift4_1:
       CFConstantStringClassName =
           Triple.isOSDarwin() ? "__T015SwiftFoundation19_NSCFConstantStringCN"
                               : "__T010Foundation19_NSCFConstantStringCN";
       Ty = IntPtrTy;
       break;
     }
 
     llvm::Constant *C = CreateRuntimeVariable(Ty, CFConstantStringClassName);
 
     if (Triple.isOSBinFormatELF() || Triple.isOSBinFormatCOFF()) {
       llvm::GlobalValue *GV = nullptr;
 
       if ((GV = dyn_cast<llvm::GlobalValue>(C))) {
         IdentifierInfo &II = Context.Idents.get(GV->getName());
         TranslationUnitDecl *TUDecl = Context.getTranslationUnitDecl();
         DeclContext *DC = TranslationUnitDecl::castToDeclContext(TUDecl);
 
         const VarDecl *VD = nullptr;
         for (const auto *Result : DC->lookup(&II))
           if ((VD = dyn_cast<VarDecl>(Result)))
             break;
 
         if (Triple.isOSBinFormatELF()) {
           if (!VD)
             GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
         } else {
           GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
           if (!VD || !VD->hasAttr<DLLExportAttr>())
             GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
           else
             GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
         }
 
         setDSOLocal(GV);
       }
     }
 
     // Decay array -> ptr
     CFConstantStringClassRef =
         IsSwiftABI ? llvm::ConstantExpr::getPtrToInt(C, Ty)
                    : llvm::ConstantExpr::getGetElementPtr(Ty, C, Zeros);
   }
 
   QualType CFTy = Context.getCFConstantStringType();
 
   auto *STy = cast<llvm::StructType>(getTypes().ConvertType(CFTy));
 
   ConstantInitBuilder Builder(*this);
   auto Fields = Builder.beginStruct(STy);
 
   // Class pointer.
   Fields.add(cast<llvm::ConstantExpr>(CFConstantStringClassRef));
 
   // Flags.
   if (IsSwiftABI) {
     Fields.addInt(IntPtrTy, IsSwift4_1 ? 0x05 : 0x01);
     Fields.addInt(Int64Ty, isUTF16 ? 0x07d0 : 0x07c8);
   } else {
     Fields.addInt(IntTy, isUTF16 ? 0x07d0 : 0x07C8);
   }
 
   // String pointer.
   llvm::Constant *C = nullptr;
   if (isUTF16) {
     auto Arr = llvm::makeArrayRef(
         reinterpret_cast<uint16_t *>(const_cast<char *>(Entry.first().data())),
         Entry.first().size() / 2);
     C = llvm::ConstantDataArray::get(VMContext, Arr);
   } else {
     C = llvm::ConstantDataArray::getString(VMContext, Entry.first());
   }
 
   // Note: -fwritable-strings doesn't make the backing store strings of
   // CFStrings writable. (See <rdar://problem/10657500>)
   auto *GV =
       new llvm::GlobalVariable(getModule(), C->getType(), /*isConstant=*/true,
                                llvm::GlobalValue::PrivateLinkage, C, ".str");
   GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
   // Don't enforce the target's minimum global alignment, since the only use
   // of the string is via this class initializer.
   CharUnits Align = isUTF16 ? Context.getTypeAlignInChars(Context.ShortTy)
                             : Context.getTypeAlignInChars(Context.CharTy);
   GV->setAlignment(Align.getAsAlign());
 
   // FIXME: We set the section explicitly to avoid a bug in ld64 224.1.
   // Without it LLVM can merge the string with a non unnamed_addr one during
   // LTO.  Doing that changes the section it ends in, which surprises ld64.
   if (Triple.isOSBinFormatMachO())
     GV->setSection(isUTF16 ? "__TEXT,__ustring"
                            : "__TEXT,__cstring,cstring_literals");
   // Make sure the literal ends up in .rodata to allow for safe ICF and for
   // the static linker to adjust permissions to read-only later on.
   else if (Triple.isOSBinFormatELF())
     GV->setSection(".rodata");
 
   // String.
   llvm::Constant *Str =
       llvm::ConstantExpr::getGetElementPtr(GV->getValueType(), GV, Zeros);
 
   if (isUTF16)
     // Cast the UTF16 string to the correct type.
     Str = llvm::ConstantExpr::getBitCast(Str, Int8PtrTy);
   Fields.add(Str);
 
   // String length.
   llvm::IntegerType *LengthTy =
       llvm::IntegerType::get(getModule().getContext(),
                              Context.getTargetInfo().getLongWidth());
   if (IsSwiftABI) {
     if (CFRuntime == LangOptions::CoreFoundationABI::Swift4_1 ||
         CFRuntime == LangOptions::CoreFoundationABI::Swift4_2)
       LengthTy = Int32Ty;
     else
       LengthTy = IntPtrTy;
   }
   Fields.addInt(LengthTy, StringLength);
 
   // Swift ABI requires 8-byte alignment to ensure that the _Atomic(uint64_t) is
   // properly aligned on 32-bit platforms.
   CharUnits Alignment =
       IsSwiftABI ? Context.toCharUnitsFromBits(64) : getPointerAlign();
 
   // The struct.
   GV = Fields.finishAndCreateGlobal("_unnamed_cfstring_", Alignment,
                                     /*isConstant=*/false,
                                     llvm::GlobalVariable::PrivateLinkage);
   GV->addAttribute("objc_arc_inert");
   switch (Triple.getObjectFormat()) {
   case llvm::Triple::UnknownObjectFormat:
     llvm_unreachable("unknown file format");
   case llvm::Triple::GOFF:
     llvm_unreachable("GOFF is not yet implemented");
   case llvm::Triple::XCOFF:
     llvm_unreachable("XCOFF is not yet implemented");
   case llvm::Triple::COFF:
   case llvm::Triple::ELF:
   case llvm::Triple::Wasm:
     GV->setSection("cfstring");
     break;
   case llvm::Triple::MachO:
     GV->setSection("__DATA,__cfstring");
     break;
   }
   Entry.second = GV;
 
   return ConstantAddress(GV, GV->getValueType(), Alignment);
 }
 
 bool CodeGenModule::getExpressionLocationsEnabled() const {
   return !CodeGenOpts.EmitCodeView || CodeGenOpts.DebugColumnInfo;
 }
 
 QualType CodeGenModule::getObjCFastEnumerationStateType() {
   if (ObjCFastEnumerationStateType.isNull()) {
     RecordDecl *D = Context.buildImplicitRecord("__objcFastEnumerationState");
     D->startDefinition();
 
     QualType FieldTypes[] = {
       Context.UnsignedLongTy,
       Context.getPointerType(Context.getObjCIdType()),
       Context.getPointerType(Context.UnsignedLongTy),
       Context.getConstantArrayType(Context.UnsignedLongTy,
                            llvm::APInt(32, 5), nullptr, ArrayType::Normal, 0)
     };
 
     for (size_t i = 0; i < 4; ++i) {
       FieldDecl *Field = FieldDecl::Create(Context,
                                            D,
                                            SourceLocation(),
                                            SourceLocation(), nullptr,
                                            FieldTypes[i], /*TInfo=*/nullptr,
                                            /*BitWidth=*/nullptr,
                                            /*Mutable=*/false,
                                            ICIS_NoInit);
       Field->setAccess(AS_public);
       D->addDecl(Field);
     }
 
     D->completeDefinition();
     ObjCFastEnumerationStateType = Context.getTagDeclType(D);
   }
 
   return ObjCFastEnumerationStateType;
 }
 
 llvm::Constant *
 CodeGenModule::GetConstantArrayFromStringLiteral(const StringLiteral *E) {
   assert(!E->getType()->isPointerType() && "Strings are always arrays");
 
   // Don't emit it as the address of the string, emit the string data itself
   // as an inline array.
   if (E->getCharByteWidth() == 1) {
     SmallString<64> Str(E->getString());
 
     // Resize the string to the right size, which is indicated by its type.
     const ConstantArrayType *CAT = Context.getAsConstantArrayType(E->getType());
     Str.resize(CAT->getSize().getZExtValue());
     return llvm::ConstantDataArray::getString(VMContext, Str, false);
   }
 
   auto *AType = cast<llvm::ArrayType>(getTypes().ConvertType(E->getType()));
   llvm::Type *ElemTy = AType->getElementType();
   unsigned NumElements = AType->getNumElements();
 
   // Wide strings have either 2-byte or 4-byte elements.
   if (ElemTy->getPrimitiveSizeInBits() == 16) {
     SmallVector<uint16_t, 32> Elements;
     Elements.reserve(NumElements);
 
     for(unsigned i = 0, e = E->getLength(); i != e; ++i)
       Elements.push_back(E->getCodeUnit(i));
     Elements.resize(NumElements);
     return llvm::ConstantDataArray::get(VMContext, Elements);
   }
 
   assert(ElemTy->getPrimitiveSizeInBits() == 32);
   SmallVector<uint32_t, 32> Elements;
   Elements.reserve(NumElements);
 
   for(unsigned i = 0, e = E->getLength(); i != e; ++i)
     Elements.push_back(E->getCodeUnit(i));
   Elements.resize(NumElements);
   return llvm::ConstantDataArray::get(VMContext, Elements);
 }
 
 static llvm::GlobalVariable *
 GenerateStringLiteral(llvm::Constant *C, llvm::GlobalValue::LinkageTypes LT,
                       CodeGenModule &CGM, StringRef GlobalName,
                       CharUnits Alignment) {
   unsigned AddrSpace = CGM.getContext().getTargetAddressSpace(
       CGM.GetGlobalConstantAddressSpace());
 
   llvm::Module &M = CGM.getModule();
   // Create a global variable for this string
   auto *GV = new llvm::GlobalVariable(
       M, C->getType(), !CGM.getLangOpts().WritableStrings, LT, C, GlobalName,
       nullptr, llvm::GlobalVariable::NotThreadLocal, AddrSpace);
   GV->setAlignment(Alignment.getAsAlign());
   GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
   if (GV->isWeakForLinker()) {
     assert(CGM.supportsCOMDAT() && "Only COFF uses weak string literals");
     GV->setComdat(M.getOrInsertComdat(GV->getName()));
   }
   CGM.setDSOLocal(GV);
 
   return GV;
 }
 
 /// GetAddrOfConstantStringFromLiteral - Return a pointer to a
 /// constant array for the given string literal.
 ConstantAddress
 CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S,
                                                   StringRef Name) {
   CharUnits Alignment = getContext().getAlignOfGlobalVarInChars(S->getType());
 
   llvm::Constant *C = GetConstantArrayFromStringLiteral(S);
   llvm::GlobalVariable **Entry = nullptr;
   if (!LangOpts.WritableStrings) {
     Entry = &ConstantStringMap[C];
     if (auto GV = *Entry) {
       if (uint64_t(Alignment.getQuantity()) > GV->getAlignment())
         GV->setAlignment(Alignment.getAsAlign());
       return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
                              GV->getValueType(), Alignment);
     }
   }
 
   SmallString<256> MangledNameBuffer;
   StringRef GlobalVariableName;
   llvm::GlobalValue::LinkageTypes LT;
 
   // Mangle the string literal if that's how the ABI merges duplicate strings.
   // Don't do it if they are writable, since we don't want writes in one TU to
   // affect strings in another.
   if (getCXXABI().getMangleContext().shouldMangleStringLiteral(S) &&
       !LangOpts.WritableStrings) {
     llvm::raw_svector_ostream Out(MangledNameBuffer);
     getCXXABI().getMangleContext().mangleStringLiteral(S, Out);
     LT = llvm::GlobalValue::LinkOnceODRLinkage;
     GlobalVariableName = MangledNameBuffer;
   } else {
     LT = llvm::GlobalValue::PrivateLinkage;
     GlobalVariableName = Name;
   }
 
   auto GV = GenerateStringLiteral(C, LT, *this, GlobalVariableName, Alignment);
   if (Entry)
     *Entry = GV;
 
   SanitizerMD->reportGlobalToASan(GV, S->getStrTokenLoc(0), "<string literal>",
                                   QualType());
 
   return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
                          GV->getValueType(), Alignment);
 }
 
 /// GetAddrOfConstantStringFromObjCEncode - Return a pointer to a constant
 /// array for the given ObjCEncodeExpr node.
 ConstantAddress
 CodeGenModule::GetAddrOfConstantStringFromObjCEncode(const ObjCEncodeExpr *E) {
   std::string Str;
   getContext().getObjCEncodingForType(E->getEncodedType(), Str);
 
   return GetAddrOfConstantCString(Str);
 }
 
 /// GetAddrOfConstantCString - Returns a pointer to a character array containing
 /// the literal and a terminating '\0' character.
 /// The result has pointer to array type.
 ConstantAddress CodeGenModule::GetAddrOfConstantCString(
     const std::string &Str, const char *GlobalName) {
   StringRef StrWithNull(Str.c_str(), Str.size() + 1);
   CharUnits Alignment =
     getContext().getAlignOfGlobalVarInChars(getContext().CharTy);
 
   llvm::Constant *C =
       llvm::ConstantDataArray::getString(getLLVMContext(), StrWithNull, false);
 
   // Don't share any string literals if strings aren't constant.
   llvm::GlobalVariable **Entry = nullptr;
   if (!LangOpts.WritableStrings) {
     Entry = &ConstantStringMap[C];
     if (auto GV = *Entry) {
       if (uint64_t(Alignment.getQuantity()) > GV->getAlignment())
         GV->setAlignment(Alignment.getAsAlign());
       return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
                              GV->getValueType(), Alignment);
     }
   }
 
   // Get the default prefix if a name wasn't specified.
   if (!GlobalName)
     GlobalName = ".str";
   // Create a global variable for this.
   auto GV = GenerateStringLiteral(C, llvm::GlobalValue::PrivateLinkage, *this,
                                   GlobalName, Alignment);
   if (Entry)
     *Entry = GV;
 
   return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
                          GV->getValueType(), Alignment);
 }
 
 ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary(
     const MaterializeTemporaryExpr *E, const Expr *Init) {
   assert((E->getStorageDuration() == SD_Static ||
           E->getStorageDuration() == SD_Thread) && "not a global temporary");
   const auto *VD = cast<VarDecl>(E->getExtendingDecl());
 
   // If we're not materializing a subobject of the temporary, keep the
   // cv-qualifiers from the type of the MaterializeTemporaryExpr.
   QualType MaterializedType = Init->getType();
   if (Init == E->getSubExpr())
     MaterializedType = E->getType();
 
   CharUnits Align = getContext().getTypeAlignInChars(MaterializedType);
 
   auto InsertResult = MaterializedGlobalTemporaryMap.insert({E, nullptr});
   if (!InsertResult.second) {
     // We've seen this before: either we already created it or we're in the
     // process of doing so.
     if (!InsertResult.first->second) {
       // We recursively re-entered this function, probably during emission of
       // the initializer. Create a placeholder. We'll clean this up in the
       // outer call, at the end of this function.
       llvm::Type *Type = getTypes().ConvertTypeForMem(MaterializedType);
       InsertResult.first->second = new llvm::GlobalVariable(
           getModule(), Type, false, llvm::GlobalVariable::InternalLinkage,
           nullptr);
     }
     return ConstantAddress(
         InsertResult.first->second,
         InsertResult.first->second->getType()->getPointerElementType(), Align);
   }
 
   // FIXME: If an externally-visible declaration extends multiple temporaries,
   // we need to give each temporary the same name in every translation unit (and
   // we also need to make the temporaries externally-visible).
   SmallString<256> Name;
   llvm::raw_svector_ostream Out(Name);
   getCXXABI().getMangleContext().mangleReferenceTemporary(
       VD, E->getManglingNumber(), Out);
 
   APValue *Value = nullptr;
   if (E->getStorageDuration() == SD_Static && VD && VD->evaluateValue()) {
     // If the initializer of the extending declaration is a constant
     // initializer, we should have a cached constant initializer for this
     // temporary. Note that this might have a different value from the value
     // computed by evaluating the initializer if the surrounding constant
     // expression modifies the temporary.
     Value = E->getOrCreateValue(false);
   }
 
   // Try evaluating it now, it might have a constant initializer.
   Expr::EvalResult EvalResult;
   if (!Value && Init->EvaluateAsRValue(EvalResult, getContext()) &&
       !EvalResult.hasSideEffects())
     Value = &EvalResult.Val;
 
   LangAS AddrSpace =
       VD ? GetGlobalVarAddressSpace(VD) : MaterializedType.getAddressSpace();
 
   Optional<ConstantEmitter> emitter;
   llvm::Constant *InitialValue = nullptr;
   bool Constant = false;
   llvm::Type *Type;
   if (Value) {
     // The temporary has a constant initializer, use it.
     emitter.emplace(*this);
     InitialValue = emitter->emitForInitializer(*Value, AddrSpace,
                                                MaterializedType);
     Constant = isTypeConstant(MaterializedType, /*ExcludeCtor*/Value);
     Type = InitialValue->getType();
   } else {
     // No initializer, the initialization will be provided when we
     // initialize the declaration which performed lifetime extension.
     Type = getTypes().ConvertTypeForMem(MaterializedType);
   }
 
   // Create a global variable for this lifetime-extended temporary.
   llvm::GlobalValue::LinkageTypes Linkage =
       getLLVMLinkageVarDefinition(VD, Constant);
   if (Linkage == llvm::GlobalVariable::ExternalLinkage) {
     const VarDecl *InitVD;
     if (VD->isStaticDataMember() && VD->getAnyInitializer(InitVD) &&
         isa<CXXRecordDecl>(InitVD->getLexicalDeclContext())) {
       // Temporaries defined inside a class get linkonce_odr linkage because the
       // class can be defined in multiple translation units.
       Linkage = llvm::GlobalVariable::LinkOnceODRLinkage;
     } else {
       // There is no need for this temporary to have external linkage if the
       // VarDecl has external linkage.
       Linkage = llvm::GlobalVariable::InternalLinkage;
     }
   }
   auto TargetAS = getContext().getTargetAddressSpace(AddrSpace);
   auto *GV = new llvm::GlobalVariable(
       getModule(), Type, Constant, Linkage, InitialValue, Name.c_str(),
       /*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal, TargetAS);
   if (emitter) emitter->finalize(GV);
   setGVProperties(GV, VD);
   GV->setAlignment(Align.getAsAlign());
   if (supportsCOMDAT() && GV->isWeakForLinker())
     GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
   if (VD->getTLSKind())
     setTLSMode(GV, *VD);
   llvm::Constant *CV = GV;
   if (AddrSpace != LangAS::Default)
     CV = getTargetCodeGenInfo().performAddrSpaceCast(
         *this, GV, AddrSpace, LangAS::Default,
         Type->getPointerTo(
             getContext().getTargetAddressSpace(LangAS::Default)));
 
   // Update the map with the new temporary. If we created a placeholder above,
   // replace it with the new global now.
   llvm::Constant *&Entry = MaterializedGlobalTemporaryMap[E];
   if (Entry) {
     Entry->replaceAllUsesWith(
         llvm::ConstantExpr::getBitCast(CV, Entry->getType()));
     llvm::cast<llvm::GlobalVariable>(Entry)->eraseFromParent();
   }
   Entry = CV;
 
   return ConstantAddress(CV, Type, Align);
 }
 
 /// EmitObjCPropertyImplementations - Emit information for synthesized
 /// properties for an implementation.
 void CodeGenModule::EmitObjCPropertyImplementations(const
                                                     ObjCImplementationDecl *D) {
   for (const auto *PID : D->property_impls()) {
     // Dynamic is just for type-checking.
     if (PID->getPropertyImplementation() == ObjCPropertyImplDecl::Synthesize) {
       ObjCPropertyDecl *PD = PID->getPropertyDecl();
 
       // Determine which methods need to be implemented, some may have
       // been overridden. Note that ::isPropertyAccessor is not the method
       // we want, that just indicates if the decl came from a
       // property. What we want to know is if the method is defined in
       // this implementation.
       auto *Getter = PID->getGetterMethodDecl();
       if (!Getter || Getter->isSynthesizedAccessorStub())
         CodeGenFunction(*this).GenerateObjCGetter(
             const_cast<ObjCImplementationDecl *>(D), PID);
       auto *Setter = PID->getSetterMethodDecl();
       if (!PD->isReadOnly() && (!Setter || Setter->isSynthesizedAccessorStub()))
         CodeGenFunction(*this).GenerateObjCSetter(
                                  const_cast<ObjCImplementationDecl *>(D), PID);
     }
   }
 }
 
 static bool needsDestructMethod(ObjCImplementationDecl *impl) {
   const ObjCInterfaceDecl *iface = impl->getClassInterface();
   for (const ObjCIvarDecl *ivar = iface->all_declared_ivar_begin();
        ivar; ivar = ivar->getNextIvar())
     if (ivar->getType().isDestructedType())
       return true;
 
   return false;
 }
 
 static bool AllTrivialInitializers(CodeGenModule &CGM,
                                    ObjCImplementationDecl *D) {
   CodeGenFunction CGF(CGM);
   for (ObjCImplementationDecl::init_iterator B = D->init_begin(),
        E = D->init_end(); B != E; ++B) {
     CXXCtorInitializer *CtorInitExp = *B;
     Expr *Init = CtorInitExp->getInit();
     if (!CGF.isTrivialInitializer(Init))
       return false;
   }
   return true;
 }
 
 /// EmitObjCIvarInitializations - Emit information for ivar initialization
 /// for an implementation.
 void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) {
   // We might need a .cxx_destruct even if we don't have any ivar initializers.
   if (needsDestructMethod(D)) {
     IdentifierInfo *II = &getContext().Idents.get(".cxx_destruct");
     Selector cxxSelector = getContext().Selectors.getSelector(0, &II);
     ObjCMethodDecl *DTORMethod = ObjCMethodDecl::Create(
         getContext(), D->getLocation(), D->getLocation(), cxxSelector,
         getContext().VoidTy, nullptr, D,
         /*isInstance=*/true, /*isVariadic=*/false,
         /*isPropertyAccessor=*/true, /*isSynthesizedAccessorStub=*/false,
         /*isImplicitlyDeclared=*/true,
         /*isDefined=*/false, ObjCMethodDecl::Required);
     D->addInstanceMethod(DTORMethod);
     CodeGenFunction(*this).GenerateObjCCtorDtorMethod(D, DTORMethod, false);
     D->setHasDestructors(true);
   }
 
   // If the implementation doesn't have any ivar initializers, we don't need
   // a .cxx_construct.
   if (D->getNumIvarInitializers() == 0 ||
       AllTrivialInitializers(*this, D))
     return;
 
   IdentifierInfo *II = &getContext().Idents.get(".cxx_construct");
   Selector cxxSelector = getContext().Selectors.getSelector(0, &II);
   // The constructor returns 'self'.
   ObjCMethodDecl *CTORMethod = ObjCMethodDecl::Create(
       getContext(), D->getLocation(), D->getLocation(), cxxSelector,
       getContext().getObjCIdType(), nullptr, D, /*isInstance=*/true,
       /*isVariadic=*/false,
       /*isPropertyAccessor=*/true, /*isSynthesizedAccessorStub=*/false,
       /*isImplicitlyDeclared=*/true,
       /*isDefined=*/false, ObjCMethodDecl::Required);
   D->addInstanceMethod(CTORMethod);
   CodeGenFunction(*this).GenerateObjCCtorDtorMethod(D, CTORMethod, true);
   D->setHasNonZeroConstructors(true);
 }
 
 // EmitLinkageSpec - Emit all declarations in a linkage spec.
 void CodeGenModule::EmitLinkageSpec(const LinkageSpecDecl *LSD) {
   if (LSD->getLanguage() != LinkageSpecDecl::lang_c &&
       LSD->getLanguage() != LinkageSpecDecl::lang_cxx) {
     ErrorUnsupported(LSD, "linkage spec");
     return;
   }
 
   EmitDeclContext(LSD);
 }
 
 void CodeGenModule::EmitDeclContext(const DeclContext *DC) {
   for (auto *I : DC->decls()) {
     // Unlike other DeclContexts, the contents of an ObjCImplDecl at TU scope
     // are themselves considered "top-level", so EmitTopLevelDecl on an
     // ObjCImplDecl does not recursively visit them. We need to do that in
     // case they're nested inside another construct (LinkageSpecDecl /
     // ExportDecl) that does stop them from being considered "top-level".
     if (auto *OID = dyn_cast<ObjCImplDecl>(I)) {
       for (auto *M : OID->methods())
         EmitTopLevelDecl(M);
     }
 
     EmitTopLevelDecl(I);
   }
 }
 
 /// EmitTopLevelDecl - Emit code for a single top level declaration.
 void CodeGenModule::EmitTopLevelDecl(Decl *D) {
   // Ignore dependent declarations.
   if (D->isTemplated())
     return;
 
   // Consteval function shouldn't be emitted.
   if (auto *FD = dyn_cast<FunctionDecl>(D))
     if (FD->isConsteval())
       return;
 
   switch (D->getKind()) {
   case Decl::CXXConversion:
   case Decl::CXXMethod:
   case Decl::Function:
     EmitGlobal(cast<FunctionDecl>(D));
     // Always provide some coverage mapping
     // even for the functions that aren't emitted.
     AddDeferredUnusedCoverageMapping(D);
     break;
 
   case Decl::CXXDeductionGuide:
     // Function-like, but does not result in code emission.
     break;
 
   case Decl::Var:
   case Decl::Decomposition:
   case Decl::VarTemplateSpecialization:
     EmitGlobal(cast<VarDecl>(D));
     if (auto *DD = dyn_cast<DecompositionDecl>(D))
       for (auto *B : DD->bindings())
         if (auto *HD = B->getHoldingVar())
           EmitGlobal(HD);
     break;
 
   // Indirect fields from global anonymous structs and unions can be
   // ignored; only the actual variable requires IR gen support.
   case Decl::IndirectField:
     break;
 
   // C++ Decls
   case Decl::Namespace:
     EmitDeclContext(cast<NamespaceDecl>(D));
     break;
   case Decl::ClassTemplateSpecialization: {
     const auto *Spec = cast<ClassTemplateSpecializationDecl>(D);
     if (CGDebugInfo *DI = getModuleDebugInfo())
       if (Spec->getSpecializationKind() ==
               TSK_ExplicitInstantiationDefinition &&
           Spec->hasDefinition())
         DI->completeTemplateDefinition(*Spec);
   } LLVM_FALLTHROUGH;
   case Decl::CXXRecord: {
     CXXRecordDecl *CRD = cast<CXXRecordDecl>(D);
     if (CGDebugInfo *DI = getModuleDebugInfo()) {
       if (CRD->hasDefinition())
         DI->EmitAndRetainType(getContext().getRecordType(cast<RecordDecl>(D)));
       if (auto *ES = D->getASTContext().getExternalSource())
         if (ES->hasExternalDefinitions(D) == ExternalASTSource::EK_Never)
           DI->completeUnusedClass(*CRD);
     }
     // Emit any static data members, they may be definitions.
     for (auto *I : CRD->decls())
       if (isa<VarDecl>(I) || isa<CXXRecordDecl>(I))
         EmitTopLevelDecl(I);
     break;
   }
     // No code generation needed.
   case Decl::UsingShadow:
   case Decl::ClassTemplate:
   case Decl::VarTemplate:
   case Decl::Concept:
   case Decl::VarTemplatePartialSpecialization:
   case Decl::FunctionTemplate:
   case Decl::TypeAliasTemplate:
   case Decl::Block:
   case Decl::Empty:
   case Decl::Binding:
     break;
   case Decl::Using:          // using X; [C++]
     if (CGDebugInfo *DI = getModuleDebugInfo())
         DI->EmitUsingDecl(cast<UsingDecl>(*D));
     break;
   case Decl::UsingEnum: // using enum X; [C++]
     if (CGDebugInfo *DI = getModuleDebugInfo())
       DI->EmitUsingEnumDecl(cast<UsingEnumDecl>(*D));
     break;
   case Decl::NamespaceAlias:
     if (CGDebugInfo *DI = getModuleDebugInfo())
         DI->EmitNamespaceAlias(cast<NamespaceAliasDecl>(*D));
     break;
   case Decl::UsingDirective: // using namespace X; [C++]
     if (CGDebugInfo *DI = getModuleDebugInfo())
       DI->EmitUsingDirective(cast<UsingDirectiveDecl>(*D));
     break;
   case Decl::CXXConstructor:
     getCXXABI().EmitCXXConstructors(cast<CXXConstructorDecl>(D));
     break;
   case Decl::CXXDestructor:
     getCXXABI().EmitCXXDestructors(cast<CXXDestructorDecl>(D));
     break;
 
   case Decl::StaticAssert:
     // Nothing to do.
     break;
 
   // Objective-C Decls
 
   // Forward declarations, no (immediate) code generation.
   case Decl::ObjCInterface:
   case Decl::ObjCCategory:
     break;
 
   case Decl::ObjCProtocol: {
     auto *Proto = cast<ObjCProtocolDecl>(D);
     if (Proto->isThisDeclarationADefinition())
       ObjCRuntime->GenerateProtocol(Proto);
     break;
   }
 
   case Decl::ObjCCategoryImpl:
     // Categories have properties but don't support synthesize so we
     // can ignore them here.
     ObjCRuntime->GenerateCategory(cast<ObjCCategoryImplDecl>(D));
     break;
 
   case Decl::ObjCImplementation: {
     auto *OMD = cast<ObjCImplementationDecl>(D);
     EmitObjCPropertyImplementations(OMD);
     EmitObjCIvarInitializations(OMD);
     ObjCRuntime->GenerateClass(OMD);
     // Emit global variable debug information.
     if (CGDebugInfo *DI = getModuleDebugInfo())
       if (getCodeGenOpts().hasReducedDebugInfo())
         DI->getOrCreateInterfaceType(getContext().getObjCInterfaceType(
             OMD->getClassInterface()), OMD->getLocation());
     break;
   }
   case Decl::ObjCMethod: {
     auto *OMD = cast<ObjCMethodDecl>(D);
     // If this is not a prototype, emit the body.
     if (OMD->getBody())
       CodeGenFunction(*this).GenerateObjCMethod(OMD);
     break;
   }
   case Decl::ObjCCompatibleAlias:
     ObjCRuntime->RegisterAlias(cast<ObjCCompatibleAliasDecl>(D));
     break;
 
   case Decl::PragmaComment: {
     const auto *PCD = cast<PragmaCommentDecl>(D);
     switch (PCD->getCommentKind()) {
     case PCK_Unknown:
       llvm_unreachable("unexpected pragma comment kind");
     case PCK_Linker:
       AppendLinkerOptions(PCD->getArg());
       break;
     case PCK_Lib:
         AddDependentLib(PCD->getArg());
       break;
     case PCK_Compiler:
     case PCK_ExeStr:
     case PCK_User:
       break; // We ignore all of these.
     }
     break;
   }
 
   case Decl::PragmaDetectMismatch: {
     const auto *PDMD = cast<PragmaDetectMismatchDecl>(D);
     AddDetectMismatch(PDMD->getName(), PDMD->getValue());
     break;
   }
 
   case Decl::LinkageSpec:
     EmitLinkageSpec(cast<LinkageSpecDecl>(D));
     break;
 
   case Decl::FileScopeAsm: {
     // File-scope asm is ignored during device-side CUDA compilation.
     if (LangOpts.CUDA && LangOpts.CUDAIsDevice)
       break;
     // File-scope asm is ignored during device-side OpenMP compilation.
     if (LangOpts.OpenMPIsDevice)
       break;
     // File-scope asm is ignored during device-side SYCL compilation.
     if (LangOpts.SYCLIsDevice)
       break;
     auto *AD = cast<FileScopeAsmDecl>(D);
     getModule().appendModuleInlineAsm(AD->getAsmString()->getString());
     break;
   }
 
   case Decl::Import: {
     auto *Import = cast<ImportDecl>(D);
 
     // If we've already imported this module, we're done.
     if (!ImportedModules.insert(Import->getImportedModule()))
       break;
 
     // Emit debug information for direct imports.
     if (!Import->getImportedOwningModule()) {
       if (CGDebugInfo *DI = getModuleDebugInfo())
         DI->EmitImportDecl(*Import);
     }
 
     // Find all of the submodules and emit the module initializers.
     llvm::SmallPtrSet<clang::Module *, 16> Visited;
     SmallVector<clang::Module *, 16> Stack;
     Visited.insert(Import->getImportedModule());
     Stack.push_back(Import->getImportedModule());
 
     while (!Stack.empty()) {
       clang::Module *Mod = Stack.pop_back_val();
       if (!EmittedModuleInitializers.insert(Mod).second)
         continue;
 
       for (auto *D : Context.getModuleInitializers(Mod))
         EmitTopLevelDecl(D);
 
       // Visit the submodules of this module.
       for (clang::Module::submodule_iterator Sub = Mod->submodule_begin(),
                                              SubEnd = Mod->submodule_end();
            Sub != SubEnd; ++Sub) {
         // Skip explicit children; they need to be explicitly imported to emit
         // the initializers.
         if ((*Sub)->IsExplicit)
           continue;
 
         if (Visited.insert(*Sub).second)
           Stack.push_back(*Sub);
       }
     }
     break;
   }
 
   case Decl::Export:
     EmitDeclContext(cast<ExportDecl>(D));
     break;
 
   case Decl::OMPThreadPrivate:
     EmitOMPThreadPrivateDecl(cast<OMPThreadPrivateDecl>(D));
     break;
 
   case Decl::OMPAllocate:
     EmitOMPAllocateDecl(cast<OMPAllocateDecl>(D));
     break;
 
   case Decl::OMPDeclareReduction:
     EmitOMPDeclareReduction(cast<OMPDeclareReductionDecl>(D));
     break;
 
   case Decl::OMPDeclareMapper:
     EmitOMPDeclareMapper(cast<OMPDeclareMapperDecl>(D));
     break;
 
   case Decl::OMPRequires:
     EmitOMPRequiresDecl(cast<OMPRequiresDecl>(D));
     break;
 
   case Decl::Typedef:
   case Decl::TypeAlias: // using foo = bar; [C++11]
     if (CGDebugInfo *DI = getModuleDebugInfo())
       DI->EmitAndRetainType(
           getContext().getTypedefType(cast<TypedefNameDecl>(D)));
     break;
 
   case Decl::Record:
     if (CGDebugInfo *DI = getModuleDebugInfo())
       if (cast<RecordDecl>(D)->getDefinition())
         DI->EmitAndRetainType(getContext().getRecordType(cast<RecordDecl>(D)));
     break;
 
   case Decl::Enum:
     if (CGDebugInfo *DI = getModuleDebugInfo())
       if (cast<EnumDecl>(D)->getDefinition())
         DI->EmitAndRetainType(getContext().getEnumType(cast<EnumDecl>(D)));
     break;
 
   default:
     // Make sure we handled everything we should, every other kind is a
     // non-top-level decl.  FIXME: Would be nice to have an isTopLevelDeclKind
     // function. Need to recode Decl::Kind to do that easily.
     assert(isa<TypeDecl>(D) && "Unsupported decl kind");
     break;
   }
 }
 
 void CodeGenModule::AddDeferredUnusedCoverageMapping(Decl *D) {
   // Do we need to generate coverage mapping?
   if (!CodeGenOpts.CoverageMapping)
     return;
   switch (D->getKind()) {
   case Decl::CXXConversion:
   case Decl::CXXMethod:
   case Decl::Function:
   case Decl::ObjCMethod:
   case Decl::CXXConstructor:
   case Decl::CXXDestructor: {
     if (!cast<FunctionDecl>(D)->doesThisDeclarationHaveABody())
       break;
     SourceManager &SM = getContext().getSourceManager();
     if (LimitedCoverage && SM.getMainFileID() != SM.getFileID(D->getBeginLoc()))
       break;
     auto I = DeferredEmptyCoverageMappingDecls.find(D);
     if (I == DeferredEmptyCoverageMappingDecls.end())
       DeferredEmptyCoverageMappingDecls[D] = true;
     break;
   }
   default:
     break;
   };
 }
 
 void CodeGenModule::ClearUnusedCoverageMapping(const Decl *D) {
   // Do we need to generate coverage mapping?
   if (!CodeGenOpts.CoverageMapping)
     return;
   if (const auto *Fn = dyn_cast<FunctionDecl>(D)) {
     if (Fn->isTemplateInstantiation())
       ClearUnusedCoverageMapping(Fn->getTemplateInstantiationPattern());
   }
   auto I = DeferredEmptyCoverageMappingDecls.find(D);
   if (I == DeferredEmptyCoverageMappingDecls.end())
     DeferredEmptyCoverageMappingDecls[D] = false;
   else
     I->second = false;
 }
 
 void CodeGenModule::EmitDeferredUnusedCoverageMappings() {
   // We call takeVector() here to avoid use-after-free.
   // FIXME: DeferredEmptyCoverageMappingDecls is getting mutated because
   // we deserialize function bodies to emit coverage info for them, and that
   // deserializes more declarations. How should we handle that case?
   for (const auto &Entry : DeferredEmptyCoverageMappingDecls.takeVector()) {
     if (!Entry.second)
       continue;
     const Decl *D = Entry.first;
     switch (D->getKind()) {
     case Decl::CXXConversion:
     case Decl::CXXMethod:
     case Decl::Function:
     case Decl::ObjCMethod: {
       CodeGenPGO PGO(*this);
       GlobalDecl GD(cast<FunctionDecl>(D));
       PGO.emitEmptyCounterMapping(D, getMangledName(GD),
                                   getFunctionLinkage(GD));
       break;
     }
     case Decl::CXXConstructor: {
       CodeGenPGO PGO(*this);
       GlobalDecl GD(cast<CXXConstructorDecl>(D), Ctor_Base);
       PGO.emitEmptyCounterMapping(D, getMangledName(GD),
                                   getFunctionLinkage(GD));
       break;
     }
     case Decl::CXXDestructor: {
       CodeGenPGO PGO(*this);
       GlobalDecl GD(cast<CXXDestructorDecl>(D), Dtor_Base);
       PGO.emitEmptyCounterMapping(D, getMangledName(GD),
                                   getFunctionLinkage(GD));
       break;
     }
     default:
       break;
     };
   }
 }
 
 void CodeGenModule::EmitMainVoidAlias() {
   // In order to transition away from "__original_main" gracefully, emit an
   // alias for "main" in the no-argument case so that libc can detect when
   // new-style no-argument main is in used.
   if (llvm::Function *F = getModule().getFunction("main")) {
     if (!F->isDeclaration() && F->arg_size() == 0 && !F->isVarArg() &&
         F->getReturnType()->isIntegerTy(Context.getTargetInfo().getIntWidth()))
       addUsedGlobal(llvm::GlobalAlias::create("__main_void", F));
   }
 }
 
 /// Turns the given pointer into a constant.
 static llvm::Constant *GetPointerConstant(llvm::LLVMContext &Context,
                                           const void *Ptr) {
   uintptr_t PtrInt = reinterpret_cast<uintptr_t>(Ptr);
   llvm::Type *i64 = llvm::Type::getInt64Ty(Context);
   return llvm::ConstantInt::get(i64, PtrInt);
 }
 
 static void EmitGlobalDeclMetadata(CodeGenModule &CGM,
                                    llvm::NamedMDNode *&GlobalMetadata,
                                    GlobalDecl D,
                                    llvm::GlobalValue *Addr) {
   if (!GlobalMetadata)
     GlobalMetadata =
       CGM.getModule().getOrInsertNamedMetadata("clang.global.decl.ptrs");
 
   // TODO: should we report variant information for ctors/dtors?
   llvm::Metadata *Ops[] = {llvm::ConstantAsMetadata::get(Addr),
                            llvm::ConstantAsMetadata::get(GetPointerConstant(
                                CGM.getLLVMContext(), D.getDecl()))};
   GlobalMetadata->addOperand(llvm::MDNode::get(CGM.getLLVMContext(), Ops));
 }
 
 /// For each function which is declared within an extern "C" region and marked
 /// as 'used', but has internal linkage, create an alias from the unmangled
 /// name to the mangled name if possible. People expect to be able to refer
 /// to such functions with an unmangled name from inline assembly within the
 /// same translation unit.
 void CodeGenModule::EmitStaticExternCAliases() {
   if (!getTargetCodeGenInfo().shouldEmitStaticExternCAliases())
     return;
   for (auto &I : StaticExternCValues) {
     IdentifierInfo *Name = I.first;
     llvm::GlobalValue *Val = I.second;
     if (Val && !getModule().getNamedValue(Name->getName()))
       addCompilerUsedGlobal(llvm::GlobalAlias::create(Name->getName(), Val));
   }
 }
 
 bool CodeGenModule::lookupRepresentativeDecl(StringRef MangledName,
                                              GlobalDecl &Result) const {
   auto Res = Manglings.find(MangledName);
   if (Res == Manglings.end())
     return false;
   Result = Res->getValue();
   return true;
 }
 
 /// Emits metadata nodes associating all the global values in the
 /// current module with the Decls they came from.  This is useful for
 /// projects using IR gen as a subroutine.
 ///
 /// Since there's currently no way to associate an MDNode directly
 /// with an llvm::GlobalValue, we create a global named metadata
 /// with the name 'clang.global.decl.ptrs'.
 void CodeGenModule::EmitDeclMetadata() {
   llvm::NamedMDNode *GlobalMetadata = nullptr;
 
   for (auto &I : MangledDeclNames) {
     llvm::GlobalValue *Addr = getModule().getNamedValue(I.second);
     // Some mangled names don't necessarily have an associated GlobalValue
     // in this module, e.g. if we mangled it for DebugInfo.
     if (Addr)
       EmitGlobalDeclMetadata(*this, GlobalMetadata, I.first, Addr);
   }
 }
 
 /// Emits metadata nodes for all the local variables in the current
 /// function.
 void CodeGenFunction::EmitDeclMetadata() {
   if (LocalDeclMap.empty()) return;
 
   llvm::LLVMContext &Context = getLLVMContext();
 
   // Find the unique metadata ID for this name.
   unsigned DeclPtrKind = Context.getMDKindID("clang.decl.ptr");
 
   llvm::NamedMDNode *GlobalMetadata = nullptr;
 
   for (auto &I : LocalDeclMap) {
     const Decl *D = I.first;
     llvm::Value *Addr = I.second.getPointer();
     if (auto *Alloca = dyn_cast<llvm::AllocaInst>(Addr)) {
       llvm::Value *DAddr = GetPointerConstant(getLLVMContext(), D);
       Alloca->setMetadata(
           DeclPtrKind, llvm::MDNode::get(
                            Context, llvm::ValueAsMetadata::getConstant(DAddr)));
     } else if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr)) {
       GlobalDecl GD = GlobalDecl(cast<VarDecl>(D));
       EmitGlobalDeclMetadata(CGM, GlobalMetadata, GD, GV);
     }
   }
 }
 
 void CodeGenModule::EmitVersionIdentMetadata() {
   llvm::NamedMDNode *IdentMetadata =
     TheModule.getOrInsertNamedMetadata("llvm.ident");
   std::string Version = getClangFullVersion();
   llvm::LLVMContext &Ctx = TheModule.getContext();
 
   llvm::Metadata *IdentNode[] = {llvm::MDString::get(Ctx, Version)};
   IdentMetadata->addOperand(llvm::MDNode::get(Ctx, IdentNode));
 }
 
 void CodeGenModule::EmitCommandLineMetadata() {
   llvm::NamedMDNode *CommandLineMetadata =
     TheModule.getOrInsertNamedMetadata("llvm.commandline");
   std::string CommandLine = getCodeGenOpts().RecordCommandLine;
   llvm::LLVMContext &Ctx = TheModule.getContext();
 
   llvm::Metadata *CommandLineNode[] = {llvm::MDString::get(Ctx, CommandLine)};
   CommandLineMetadata->addOperand(llvm::MDNode::get(Ctx, CommandLineNode));
 }
 
 void CodeGenModule::EmitCoverageFile() {
   if (getCodeGenOpts().CoverageDataFile.empty() &&
       getCodeGenOpts().CoverageNotesFile.empty())
     return;
 
   llvm::NamedMDNode *CUNode = TheModule.getNamedMetadata("llvm.dbg.cu");
   if (!CUNode)
     return;
 
   llvm::NamedMDNode *GCov = TheModule.getOrInsertNamedMetadata("llvm.gcov");
   llvm::LLVMContext &Ctx = TheModule.getContext();
   auto *CoverageDataFile =
       llvm::MDString::get(Ctx, getCodeGenOpts().CoverageDataFile);
   auto *CoverageNotesFile =
       llvm::MDString::get(Ctx, getCodeGenOpts().CoverageNotesFile);
   for (int i = 0, e = CUNode->getNumOperands(); i != e; ++i) {
     llvm::MDNode *CU = CUNode->getOperand(i);
     llvm::Metadata *Elts[] = {CoverageNotesFile, CoverageDataFile, CU};
     GCov->addOperand(llvm::MDNode::get(Ctx, Elts));
   }
 }
 
 llvm::Constant *CodeGenModule::GetAddrOfRTTIDescriptor(QualType Ty,
                                                        bool ForEH) {
   // Return a bogus pointer if RTTI is disabled, unless it's for EH.
   // FIXME: should we even be calling this method if RTTI is disabled
   // and it's not for EH?
   if ((!ForEH && !getLangOpts().RTTI) || getLangOpts().CUDAIsDevice ||
       (getLangOpts().OpenMP && getLangOpts().OpenMPIsDevice &&
        getTriple().isNVPTX()))
     return llvm::Constant::getNullValue(Int8PtrTy);
 
   if (ForEH && Ty->isObjCObjectPointerType() &&
       LangOpts.ObjCRuntime.isGNUFamily())
     return ObjCRuntime->GetEHType(Ty);
 
   return getCXXABI().getAddrOfRTTIDescriptor(Ty);
 }
 
 void CodeGenModule::EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D) {
   // Do not emit threadprivates in simd-only mode.
   if (LangOpts.OpenMP && LangOpts.OpenMPSimd)
     return;
   for (auto RefExpr : D->varlists()) {
     auto *VD = cast<VarDecl>(cast<DeclRefExpr>(RefExpr)->getDecl());
     bool PerformInit =
         VD->getAnyInitializer() &&
         !VD->getAnyInitializer()->isConstantInitializer(getContext(),
                                                         /*ForRef=*/false);
 
     Address Addr(GetAddrOfGlobalVar(VD), getContext().getDeclAlign(VD));
     if (auto InitFunction = getOpenMPRuntime().emitThreadPrivateVarDefinition(
             VD, Addr, RefExpr->getBeginLoc(), PerformInit))
       CXXGlobalInits.push_back(InitFunction);
   }
 }
 
 llvm::Metadata *
 CodeGenModule::CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map,
                                             StringRef Suffix) {
   if (auto *FnType = T->getAs<FunctionProtoType>())
     T = getContext().getFunctionType(
         FnType->getReturnType(), FnType->getParamTypes(),
         FnType->getExtProtoInfo().withExceptionSpec(EST_None));
 
   llvm::Metadata *&InternalId = Map[T.getCanonicalType()];
   if (InternalId)
     return InternalId;
 
   if (isExternallyVisible(T->getLinkage())) {
     std::string OutName;
     llvm::raw_string_ostream Out(OutName);
     getCXXABI().getMangleContext().mangleTypeName(T, Out);
     Out << Suffix;
 
     InternalId = llvm::MDString::get(getLLVMContext(), Out.str());
   } else {
     InternalId = llvm::MDNode::getDistinct(getLLVMContext(),
                                            llvm::ArrayRef<llvm::Metadata *>());
   }
 
   return InternalId;
 }
 
 llvm::Metadata *CodeGenModule::CreateMetadataIdentifierForType(QualType T) {
   return CreateMetadataIdentifierImpl(T, MetadataIdMap, "");
 }
 
 llvm::Metadata *
 CodeGenModule::CreateMetadataIdentifierForVirtualMemPtrType(QualType T) {
   return CreateMetadataIdentifierImpl(T, VirtualMetadataIdMap, ".virtual");
 }
 
 // Generalize pointer types to a void pointer with the qualifiers of the
 // originally pointed-to type, e.g. 'const char *' and 'char * const *'
 // generalize to 'const void *' while 'char *' and 'const char **' generalize to
 // 'void *'.
 static QualType GeneralizeType(ASTContext &Ctx, QualType Ty) {
   if (!Ty->isPointerType())
     return Ty;
 
   return Ctx.getPointerType(
       QualType(Ctx.VoidTy).withCVRQualifiers(
           Ty->getPointeeType().getCVRQualifiers()));
 }
 
 // Apply type generalization to a FunctionType's return and argument types
 static QualType GeneralizeFunctionType(ASTContext &Ctx, QualType Ty) {
   if (auto *FnType = Ty->getAs<FunctionProtoType>()) {
     SmallVector<QualType, 8> GeneralizedParams;
     for (auto &Param : FnType->param_types())
       GeneralizedParams.push_back(GeneralizeType(Ctx, Param));
 
     return Ctx.getFunctionType(
         GeneralizeType(Ctx, FnType->getReturnType()),
         GeneralizedParams, FnType->getExtProtoInfo());
   }
 
   if (auto *FnType = Ty->getAs<FunctionNoProtoType>())
     return Ctx.getFunctionNoProtoType(
         GeneralizeType(Ctx, FnType->getReturnType()));
 
   llvm_unreachable("Encountered unknown FunctionType");
 }
 
 llvm::Metadata *CodeGenModule::CreateMetadataIdentifierGeneralized(QualType T) {
   return CreateMetadataIdentifierImpl(GeneralizeFunctionType(getContext(), T),
                                       GeneralizedMetadataIdMap, ".generalized");
 }
 
 /// Returns whether this module needs the "all-vtables" type identifier.
 bool CodeGenModule::NeedAllVtablesTypeId() const {
   // Returns true if at least one of vtable-based CFI checkers is enabled and
   // is not in the trapping mode.
   return ((LangOpts.Sanitize.has(SanitizerKind::CFIVCall) &&
            !CodeGenOpts.SanitizeTrap.has(SanitizerKind::CFIVCall)) ||
           (LangOpts.Sanitize.has(SanitizerKind::CFINVCall) &&
            !CodeGenOpts.SanitizeTrap.has(SanitizerKind::CFINVCall)) ||
           (LangOpts.Sanitize.has(SanitizerKind::CFIDerivedCast) &&
            !CodeGenOpts.SanitizeTrap.has(SanitizerKind::CFIDerivedCast)) ||
           (LangOpts.Sanitize.has(SanitizerKind::CFIUnrelatedCast) &&
            !CodeGenOpts.SanitizeTrap.has(SanitizerKind::CFIUnrelatedCast)));
 }
 
 void CodeGenModule::AddVTableTypeMetadata(llvm::GlobalVariable *VTable,
                                           CharUnits Offset,
                                           const CXXRecordDecl *RD) {
   llvm::Metadata *MD =
       CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
   VTable->addTypeMetadata(Offset.getQuantity(), MD);
 
   if (CodeGenOpts.SanitizeCfiCrossDso)
     if (auto CrossDsoTypeId = CreateCrossDsoCfiTypeId(MD))
       VTable->addTypeMetadata(Offset.getQuantity(),
                               llvm::ConstantAsMetadata::get(CrossDsoTypeId));
 
   if (NeedAllVtablesTypeId()) {
     llvm::Metadata *MD = llvm::MDString::get(getLLVMContext(), "all-vtables");
     VTable->addTypeMetadata(Offset.getQuantity(), MD);
   }
 }
 
 llvm::SanitizerStatReport &CodeGenModule::getSanStats() {
   if (!SanStats)
     SanStats = std::make_unique<llvm::SanitizerStatReport>(&getModule());
 
   return *SanStats;
 }
 
 llvm::Value *
 CodeGenModule::createOpenCLIntToSamplerConversion(const Expr *E,
                                                   CodeGenFunction &CGF) {
   llvm::Constant *C = ConstantEmitter(CGF).emitAbstract(E, E->getType());
   auto *SamplerT = getOpenCLRuntime().getSamplerType(E->getType().getTypePtr());
   auto *FTy = llvm::FunctionType::get(SamplerT, {C->getType()}, false);
   auto *Call = CGF.EmitRuntimeCall(
       CreateRuntimeFunction(FTy, "__translate_sampler_initializer"), {C});
   return Call;
 }
 
 CharUnits CodeGenModule::getNaturalPointeeTypeAlignment(
     QualType T, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo) {
   return getNaturalTypeAlignment(T->getPointeeType(), BaseInfo, TBAAInfo,
                                  /* forPointeeType= */ true);
 }
 
 CharUnits CodeGenModule::getNaturalTypeAlignment(QualType T,
                                                  LValueBaseInfo *BaseInfo,
                                                  TBAAAccessInfo *TBAAInfo,
                                                  bool forPointeeType) {
   if (TBAAInfo)
     *TBAAInfo = getTBAAAccessInfo(T);
 
   // FIXME: This duplicates logic in ASTContext::getTypeAlignIfKnown. But
   // that doesn't return the information we need to compute BaseInfo.
 
   // Honor alignment typedef attributes even on incomplete types.
   // We also honor them straight for C++ class types, even as pointees;
   // there's an expressivity gap here.
   if (auto TT = T->getAs<TypedefType>()) {
     if (auto Align = TT->getDecl()->getMaxAlignment()) {
       if (BaseInfo)
         *BaseInfo = LValueBaseInfo(AlignmentSource::AttributedType);
       return getContext().toCharUnitsFromBits(Align);
     }
   }
 
   bool AlignForArray = T->isArrayType();
 
   // Analyze the base element type, so we don't get confused by incomplete
   // array types.
   T = getContext().getBaseElementType(T);
 
   if (T->isIncompleteType()) {
     // We could try to replicate the logic from
     // ASTContext::getTypeAlignIfKnown, but nothing uses the alignment if the
     // type is incomplete, so it's impossible to test. We could try to reuse
     // getTypeAlignIfKnown, but that doesn't return the information we need
     // to set BaseInfo.  So just ignore the possibility that the alignment is
     // greater than one.
     if (BaseInfo)
       *BaseInfo = LValueBaseInfo(AlignmentSource::Type);
     return CharUnits::One();
   }
 
   if (BaseInfo)
     *BaseInfo = LValueBaseInfo(AlignmentSource::Type);
 
   CharUnits Alignment;
   const CXXRecordDecl *RD;
   if (T.getQualifiers().hasUnaligned()) {
     Alignment = CharUnits::One();
   } else if (forPointeeType && !AlignForArray &&
              (RD = T->getAsCXXRecordDecl())) {
     // For C++ class pointees, we don't know whether we're pointing at a
     // base or a complete object, so we generally need to use the
     // non-virtual alignment.
     Alignment = getClassPointerAlignment(RD);
   } else {
     Alignment = getContext().getTypeAlignInChars(T);
   }
 
   // Cap to the global maximum type alignment unless the alignment
   // was somehow explicit on the type.
   if (unsigned MaxAlign = getLangOpts().MaxTypeAlign) {
     if (Alignment.getQuantity() > MaxAlign &&
         !getContext().isAlignmentRequired(T))
       Alignment = CharUnits::fromQuantity(MaxAlign);
   }
   return Alignment;
 }
 
 bool CodeGenModule::stopAutoInit() {
   unsigned StopAfter = getContext().getLangOpts().TrivialAutoVarInitStopAfter;
   if (StopAfter) {
     // This number is positive only when -ftrivial-auto-var-init-stop-after=* is
     // used
     if (NumAutoVarInit >= StopAfter) {
       return true;
     }
     if (!NumAutoVarInit) {
       unsigned DiagID = getDiags().getCustomDiagID(
           DiagnosticsEngine::Warning,
           "-ftrivial-auto-var-init-stop-after=%0 has been enabled to limit the "
           "number of times ftrivial-auto-var-init=%1 gets applied.");
       getDiags().Report(DiagID)
           << StopAfter
           << (getContext().getLangOpts().getTrivialAutoVarInit() ==
                       LangOptions::TrivialAutoVarInitKind::Zero
                   ? "zero"
                   : "pattern");
     }
     ++NumAutoVarInit;
   }
   return false;
 }
 
-void CodeGenModule::printPostfixForExternalizedStaticVar(
-    llvm::raw_ostream &OS) const {
-  OS << "__static__" << getContext().getCUIDHash();
+void CodeGenModule::printPostfixForExternalizedDecl(llvm::raw_ostream &OS,
+                                                    const Decl *D) const {
+  StringRef Tag;
+  // ptxas does not allow '.' in symbol names. On the other hand, HIP prefers
+  // postfix beginning with '.' since the symbol name can be demangled.
+  if (LangOpts.HIP)
+    Tag = (isa<VarDecl>(D) ? ".static." : ".intern.");
+  else
+    Tag = (isa<VarDecl>(D) ? "__static__" : "__intern__");
+  OS << Tag << getContext().getCUIDHash();
 }
diff --git a/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.h b/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.h
index 1fcd5d4d808a..a8a63c8da57f 100644
--- a/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.h
+++ b/contrib/llvm-project/clang/lib/CodeGen/CodeGenModule.h
@@ -1,1622 +1,1623 @@
 //===--- CodeGenModule.h - Per-Module state for LLVM CodeGen ----*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This is the internal per-translation-unit state used for llvm translation.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENMODULE_H
 #define LLVM_CLANG_LIB_CODEGEN_CODEGENMODULE_H
 
 #include "CGVTables.h"
 #include "CodeGenTypeCache.h"
 #include "CodeGenTypes.h"
 #include "SanitizerMetadata.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclOpenMP.h"
 #include "clang/AST/GlobalDecl.h"
 #include "clang/AST/Mangle.h"
 #include "clang/Basic/ABI.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/Module.h"
 #include "clang/Basic/NoSanitizeList.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/XRayLists.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Transforms/Utils/SanitizerStats.h"
 
 namespace llvm {
 class Module;
 class Constant;
 class ConstantInt;
 class Function;
 class GlobalValue;
 class DataLayout;
 class FunctionType;
 class LLVMContext;
 class IndexedInstrProfReader;
 }
 
 namespace clang {
 class ASTContext;
 class AtomicType;
 class FunctionDecl;
 class IdentifierInfo;
 class ObjCImplementationDecl;
 class ObjCEncodeExpr;
 class BlockExpr;
 class CharUnits;
 class Decl;
 class Expr;
 class Stmt;
 class StringLiteral;
 class NamedDecl;
 class ValueDecl;
 class VarDecl;
 class LangOptions;
 class CodeGenOptions;
 class HeaderSearchOptions;
 class DiagnosticsEngine;
 class AnnotateAttr;
 class CXXDestructorDecl;
 class Module;
 class CoverageSourceInfo;
 class InitSegAttr;
 
 namespace CodeGen {
 
 class CodeGenFunction;
 class CodeGenTBAA;
 class CGCXXABI;
 class CGDebugInfo;
 class CGObjCRuntime;
 class CGOpenCLRuntime;
 class CGOpenMPRuntime;
 class CGCUDARuntime;
 class CoverageMappingModuleGen;
 class TargetCodeGenInfo;
 
 enum ForDefinition_t : bool {
   NotForDefinition = false,
   ForDefinition = true
 };
 
 struct OrderGlobalInitsOrStermFinalizers {
   unsigned int priority;
   unsigned int lex_order;
   OrderGlobalInitsOrStermFinalizers(unsigned int p, unsigned int l)
       : priority(p), lex_order(l) {}
 
   bool operator==(const OrderGlobalInitsOrStermFinalizers &RHS) const {
     return priority == RHS.priority && lex_order == RHS.lex_order;
   }
 
   bool operator<(const OrderGlobalInitsOrStermFinalizers &RHS) const {
     return std::tie(priority, lex_order) <
            std::tie(RHS.priority, RHS.lex_order);
   }
 };
 
 struct ObjCEntrypoints {
   ObjCEntrypoints() { memset(this, 0, sizeof(*this)); }
 
   /// void objc_alloc(id);
   llvm::FunctionCallee objc_alloc;
 
   /// void objc_allocWithZone(id);
   llvm::FunctionCallee objc_allocWithZone;
 
   /// void objc_alloc_init(id);
   llvm::FunctionCallee objc_alloc_init;
 
   /// void objc_autoreleasePoolPop(void*);
   llvm::FunctionCallee objc_autoreleasePoolPop;
 
   /// void objc_autoreleasePoolPop(void*);
   /// Note this method is used when we are using exception handling
   llvm::FunctionCallee objc_autoreleasePoolPopInvoke;
 
   /// void *objc_autoreleasePoolPush(void);
   llvm::Function *objc_autoreleasePoolPush;
 
   /// id objc_autorelease(id);
   llvm::Function *objc_autorelease;
 
   /// id objc_autorelease(id);
   /// Note this is the runtime method not the intrinsic.
   llvm::FunctionCallee objc_autoreleaseRuntimeFunction;
 
   /// id objc_autoreleaseReturnValue(id);
   llvm::Function *objc_autoreleaseReturnValue;
 
   /// void objc_copyWeak(id *dest, id *src);
   llvm::Function *objc_copyWeak;
 
   /// void objc_destroyWeak(id*);
   llvm::Function *objc_destroyWeak;
 
   /// id objc_initWeak(id*, id);
   llvm::Function *objc_initWeak;
 
   /// id objc_loadWeak(id*);
   llvm::Function *objc_loadWeak;
 
   /// id objc_loadWeakRetained(id*);
   llvm::Function *objc_loadWeakRetained;
 
   /// void objc_moveWeak(id *dest, id *src);
   llvm::Function *objc_moveWeak;
 
   /// id objc_retain(id);
   llvm::Function *objc_retain;
 
   /// id objc_retain(id);
   /// Note this is the runtime method not the intrinsic.
   llvm::FunctionCallee objc_retainRuntimeFunction;
 
   /// id objc_retainAutorelease(id);
   llvm::Function *objc_retainAutorelease;
 
   /// id objc_retainAutoreleaseReturnValue(id);
   llvm::Function *objc_retainAutoreleaseReturnValue;
 
   /// id objc_retainAutoreleasedReturnValue(id);
   llvm::Function *objc_retainAutoreleasedReturnValue;
 
   /// id objc_retainBlock(id);
   llvm::Function *objc_retainBlock;
 
   /// void objc_release(id);
   llvm::Function *objc_release;
 
   /// void objc_release(id);
   /// Note this is the runtime method not the intrinsic.
   llvm::FunctionCallee objc_releaseRuntimeFunction;
 
   /// void objc_storeStrong(id*, id);
   llvm::Function *objc_storeStrong;
 
   /// id objc_storeWeak(id*, id);
   llvm::Function *objc_storeWeak;
 
   /// id objc_unsafeClaimAutoreleasedReturnValue(id);
   llvm::Function *objc_unsafeClaimAutoreleasedReturnValue;
 
   /// A void(void) inline asm to use to mark that the return value of
   /// a call will be immediately retain.
   llvm::InlineAsm *retainAutoreleasedReturnValueMarker;
 
   /// void clang.arc.use(...);
   llvm::Function *clang_arc_use;
 
   /// void clang.arc.noop.use(...);
   llvm::Function *clang_arc_noop_use;
 };
 
 /// This class records statistics on instrumentation based profiling.
 class InstrProfStats {
   uint32_t VisitedInMainFile;
   uint32_t MissingInMainFile;
   uint32_t Visited;
   uint32_t Missing;
   uint32_t Mismatched;
 
 public:
   InstrProfStats()
       : VisitedInMainFile(0), MissingInMainFile(0), Visited(0), Missing(0),
         Mismatched(0) {}
   /// Record that we've visited a function and whether or not that function was
   /// in the main source file.
   void addVisited(bool MainFile) {
     if (MainFile)
       ++VisitedInMainFile;
     ++Visited;
   }
   /// Record that a function we've visited has no profile data.
   void addMissing(bool MainFile) {
     if (MainFile)
       ++MissingInMainFile;
     ++Missing;
   }
   /// Record that a function we've visited has mismatched profile data.
   void addMismatched(bool MainFile) { ++Mismatched; }
   /// Whether or not the stats we've gathered indicate any potential problems.
   bool hasDiagnostics() { return Missing || Mismatched; }
   /// Report potential problems we've found to \c Diags.
   void reportDiagnostics(DiagnosticsEngine &Diags, StringRef MainFile);
 };
 
 /// A pair of helper functions for a __block variable.
 class BlockByrefHelpers : public llvm::FoldingSetNode {
   // MSVC requires this type to be complete in order to process this
   // header.
 public:
   llvm::Constant *CopyHelper;
   llvm::Constant *DisposeHelper;
 
   /// The alignment of the field.  This is important because
   /// different offsets to the field within the byref struct need to
   /// have different helper functions.
   CharUnits Alignment;
 
   BlockByrefHelpers(CharUnits alignment)
       : CopyHelper(nullptr), DisposeHelper(nullptr), Alignment(alignment) {}
   BlockByrefHelpers(const BlockByrefHelpers &) = default;
   virtual ~BlockByrefHelpers();
 
   void Profile(llvm::FoldingSetNodeID &id) const {
     id.AddInteger(Alignment.getQuantity());
     profileImpl(id);
   }
   virtual void profileImpl(llvm::FoldingSetNodeID &id) const = 0;
 
   virtual bool needsCopy() const { return true; }
   virtual void emitCopy(CodeGenFunction &CGF, Address dest, Address src) = 0;
 
   virtual bool needsDispose() const { return true; }
   virtual void emitDispose(CodeGenFunction &CGF, Address field) = 0;
 };
 
 /// This class organizes the cross-function state that is used while generating
 /// LLVM code.
 class CodeGenModule : public CodeGenTypeCache {
   CodeGenModule(const CodeGenModule &) = delete;
   void operator=(const CodeGenModule &) = delete;
 
 public:
   struct Structor {
     Structor() : Priority(0), Initializer(nullptr), AssociatedData(nullptr) {}
     Structor(int Priority, llvm::Constant *Initializer,
              llvm::Constant *AssociatedData)
         : Priority(Priority), Initializer(Initializer),
           AssociatedData(AssociatedData) {}
     int Priority;
     llvm::Constant *Initializer;
     llvm::Constant *AssociatedData;
   };
 
   typedef std::vector<Structor> CtorList;
 
 private:
   ASTContext &Context;
   const LangOptions &LangOpts;
   const HeaderSearchOptions &HeaderSearchOpts; // Only used for debug info.
   const PreprocessorOptions &PreprocessorOpts; // Only used for debug info.
   const CodeGenOptions &CodeGenOpts;
   unsigned NumAutoVarInit = 0;
   llvm::Module &TheModule;
   DiagnosticsEngine &Diags;
   const TargetInfo &Target;
   std::unique_ptr<CGCXXABI> ABI;
   llvm::LLVMContext &VMContext;
   std::string ModuleNameHash;
 
   std::unique_ptr<CodeGenTBAA> TBAA;
 
   mutable std::unique_ptr<TargetCodeGenInfo> TheTargetCodeGenInfo;
 
   // This should not be moved earlier, since its initialization depends on some
   // of the previous reference members being already initialized and also checks
   // if TheTargetCodeGenInfo is NULL
   CodeGenTypes Types;
 
   /// Holds information about C++ vtables.
   CodeGenVTables VTables;
 
   std::unique_ptr<CGObjCRuntime> ObjCRuntime;
   std::unique_ptr<CGOpenCLRuntime> OpenCLRuntime;
   std::unique_ptr<CGOpenMPRuntime> OpenMPRuntime;
   std::unique_ptr<CGCUDARuntime> CUDARuntime;
   std::unique_ptr<CGDebugInfo> DebugInfo;
   std::unique_ptr<ObjCEntrypoints> ObjCData;
   llvm::MDNode *NoObjCARCExceptionsMetadata = nullptr;
   std::unique_ptr<llvm::IndexedInstrProfReader> PGOReader;
   InstrProfStats PGOStats;
   std::unique_ptr<llvm::SanitizerStatReport> SanStats;
 
   // A set of references that have only been seen via a weakref so far. This is
   // used to remove the weak of the reference if we ever see a direct reference
   // or a definition.
   llvm::SmallPtrSet<llvm::GlobalValue*, 10> WeakRefReferences;
 
   /// This contains all the decls which have definitions but/ which are deferred
   /// for emission and therefore should only be output if they are actually
   /// used. If a decl is in this, then it is known to have not been referenced
   /// yet.
   llvm::DenseMap<StringRef, GlobalDecl> DeferredDecls;
 
   /// This is a list of deferred decls which we have seen that *are* actually
   /// referenced. These get code generated when the module is done.
   std::vector<GlobalDecl> DeferredDeclsToEmit;
   void addDeferredDeclToEmit(GlobalDecl GD) {
     DeferredDeclsToEmit.emplace_back(GD);
   }
 
   /// List of alias we have emitted. Used to make sure that what they point to
   /// is defined once we get to the end of the of the translation unit.
   std::vector<GlobalDecl> Aliases;
 
   /// List of multiversion functions that have to be emitted.  Used to make sure
   /// we properly emit the iFunc.
   std::vector<GlobalDecl> MultiVersionFuncs;
 
   typedef llvm::StringMap<llvm::TrackingVH<llvm::Constant> > ReplacementsTy;
   ReplacementsTy Replacements;
 
   /// List of global values to be replaced with something else. Used when we
   /// want to replace a GlobalValue but can't identify it by its mangled name
   /// anymore (because the name is already taken).
   llvm::SmallVector<std::pair<llvm::GlobalValue *, llvm::Constant *>, 8>
     GlobalValReplacements;
 
   /// Variables for which we've emitted globals containing their constant
   /// values along with the corresponding globals, for opportunistic reuse.
   llvm::DenseMap<const VarDecl*, llvm::GlobalVariable*> InitializerConstants;
 
   /// Set of global decls for which we already diagnosed mangled name conflict.
   /// Required to not issue a warning (on a mangling conflict) multiple times
   /// for the same decl.
   llvm::DenseSet<GlobalDecl> DiagnosedConflictingDefinitions;
 
   /// A queue of (optional) vtables to consider emitting.
   std::vector<const CXXRecordDecl*> DeferredVTables;
 
   /// A queue of (optional) vtables that may be emitted opportunistically.
   std::vector<const CXXRecordDecl *> OpportunisticVTables;
 
   /// List of global values which are required to be present in the object file;
   /// bitcast to i8*. This is used for forcing visibility of symbols which may
   /// otherwise be optimized out.
   std::vector<llvm::WeakTrackingVH> LLVMUsed;
   std::vector<llvm::WeakTrackingVH> LLVMCompilerUsed;
 
   /// Store the list of global constructors and their respective priorities to
   /// be emitted when the translation unit is complete.
   CtorList GlobalCtors;
 
   /// Store the list of global destructors and their respective priorities to be
   /// emitted when the translation unit is complete.
   CtorList GlobalDtors;
 
   /// An ordered map of canonical GlobalDecls to their mangled names.
   llvm::MapVector<GlobalDecl, StringRef> MangledDeclNames;
   llvm::StringMap<GlobalDecl, llvm::BumpPtrAllocator> Manglings;
 
   /// Global annotations.
   std::vector<llvm::Constant*> Annotations;
 
   /// Map used to get unique annotation strings.
   llvm::StringMap<llvm::Constant*> AnnotationStrings;
 
   /// Used for uniquing of annotation arguments.
   llvm::DenseMap<unsigned, llvm::Constant *> AnnotationArgs;
 
   llvm::StringMap<llvm::GlobalVariable *> CFConstantStringMap;
 
   llvm::DenseMap<llvm::Constant *, llvm::GlobalVariable *> ConstantStringMap;
   llvm::DenseMap<const Decl*, llvm::Constant *> StaticLocalDeclMap;
   llvm::DenseMap<const Decl*, llvm::GlobalVariable*> StaticLocalDeclGuardMap;
   llvm::DenseMap<const Expr*, llvm::Constant *> MaterializedGlobalTemporaryMap;
 
   llvm::DenseMap<QualType, llvm::Constant *> AtomicSetterHelperFnMap;
   llvm::DenseMap<QualType, llvm::Constant *> AtomicGetterHelperFnMap;
 
   /// Map used to get unique type descriptor constants for sanitizers.
   llvm::DenseMap<QualType, llvm::Constant *> TypeDescriptorMap;
 
   /// Map used to track internal linkage functions declared within
   /// extern "C" regions.
   typedef llvm::MapVector<IdentifierInfo *,
                           llvm::GlobalValue *> StaticExternCMap;
   StaticExternCMap StaticExternCValues;
 
   /// thread_local variables defined or used in this TU.
   std::vector<const VarDecl *> CXXThreadLocals;
 
   /// thread_local variables with initializers that need to run
   /// before any thread_local variable in this TU is odr-used.
   std::vector<llvm::Function *> CXXThreadLocalInits;
   std::vector<const VarDecl *> CXXThreadLocalInitVars;
 
   /// Global variables with initializers that need to run before main.
   std::vector<llvm::Function *> CXXGlobalInits;
 
   /// When a C++ decl with an initializer is deferred, null is
   /// appended to CXXGlobalInits, and the index of that null is placed
   /// here so that the initializer will be performed in the correct
   /// order. Once the decl is emitted, the index is replaced with ~0U to ensure
   /// that we don't re-emit the initializer.
   llvm::DenseMap<const Decl*, unsigned> DelayedCXXInitPosition;
 
   typedef std::pair<OrderGlobalInitsOrStermFinalizers, llvm::Function *>
       GlobalInitData;
 
   struct GlobalInitPriorityCmp {
     bool operator()(const GlobalInitData &LHS,
                     const GlobalInitData &RHS) const {
       return LHS.first.priority < RHS.first.priority;
     }
   };
 
   /// Global variables with initializers whose order of initialization is set by
   /// init_priority attribute.
   SmallVector<GlobalInitData, 8> PrioritizedCXXGlobalInits;
 
   /// Global destructor functions and arguments that need to run on termination.
   /// When UseSinitAndSterm is set, it instead contains sterm finalizer
   /// functions, which also run on unloading a shared library.
   typedef std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
                      llvm::Constant *>
       CXXGlobalDtorsOrStermFinalizer_t;
   SmallVector<CXXGlobalDtorsOrStermFinalizer_t, 8>
       CXXGlobalDtorsOrStermFinalizers;
 
   typedef std::pair<OrderGlobalInitsOrStermFinalizers, llvm::Function *>
       StermFinalizerData;
 
   struct StermFinalizerPriorityCmp {
     bool operator()(const StermFinalizerData &LHS,
                     const StermFinalizerData &RHS) const {
       return LHS.first.priority < RHS.first.priority;
     }
   };
 
   /// Global variables with sterm finalizers whose order of initialization is
   /// set by init_priority attribute.
   SmallVector<StermFinalizerData, 8> PrioritizedCXXStermFinalizers;
 
   /// The complete set of modules that has been imported.
   llvm::SetVector<clang::Module *> ImportedModules;
 
   /// The set of modules for which the module initializers
   /// have been emitted.
   llvm::SmallPtrSet<clang::Module *, 16> EmittedModuleInitializers;
 
   /// A vector of metadata strings for linker options.
   SmallVector<llvm::MDNode *, 16> LinkerOptionsMetadata;
 
   /// A vector of metadata strings for dependent libraries for ELF.
   SmallVector<llvm::MDNode *, 16> ELFDependentLibraries;
 
   /// @name Cache for Objective-C runtime types
   /// @{
 
   /// Cached reference to the class for constant strings. This value has type
   /// int * but is actually an Obj-C class pointer.
   llvm::WeakTrackingVH CFConstantStringClassRef;
 
   /// The type used to describe the state of a fast enumeration in
   /// Objective-C's for..in loop.
   QualType ObjCFastEnumerationStateType;
 
   /// @}
 
   /// Lazily create the Objective-C runtime
   void createObjCRuntime();
 
   void createOpenCLRuntime();
   void createOpenMPRuntime();
   void createCUDARuntime();
 
   bool isTriviallyRecursive(const FunctionDecl *F);
   bool shouldEmitFunction(GlobalDecl GD);
   bool shouldOpportunisticallyEmitVTables();
   /// Map used to be sure we don't emit the same CompoundLiteral twice.
   llvm::DenseMap<const CompoundLiteralExpr *, llvm::GlobalVariable *>
       EmittedCompoundLiterals;
 
   /// Map of the global blocks we've emitted, so that we don't have to re-emit
   /// them if the constexpr evaluator gets aggressive.
   llvm::DenseMap<const BlockExpr *, llvm::Constant *> EmittedGlobalBlocks;
 
   /// @name Cache for Blocks Runtime Globals
   /// @{
 
   llvm::Constant *NSConcreteGlobalBlock = nullptr;
   llvm::Constant *NSConcreteStackBlock = nullptr;
 
   llvm::FunctionCallee BlockObjectAssign = nullptr;
   llvm::FunctionCallee BlockObjectDispose = nullptr;
 
   llvm::Type *BlockDescriptorType = nullptr;
   llvm::Type *GenericBlockLiteralType = nullptr;
 
   struct {
     int GlobalUniqueCount;
   } Block;
 
   GlobalDecl initializedGlobalDecl;
 
   /// @}
 
   /// void @llvm.lifetime.start(i64 %size, i8* nocapture <ptr>)
   llvm::Function *LifetimeStartFn = nullptr;
 
   /// void @llvm.lifetime.end(i64 %size, i8* nocapture <ptr>)
   llvm::Function *LifetimeEndFn = nullptr;
 
   std::unique_ptr<SanitizerMetadata> SanitizerMD;
 
   llvm::MapVector<const Decl *, bool> DeferredEmptyCoverageMappingDecls;
 
   std::unique_ptr<CoverageMappingModuleGen> CoverageMapping;
 
   /// Mapping from canonical types to their metadata identifiers. We need to
   /// maintain this mapping because identifiers may be formed from distinct
   /// MDNodes.
   typedef llvm::DenseMap<QualType, llvm::Metadata *> MetadataTypeMap;
   MetadataTypeMap MetadataIdMap;
   MetadataTypeMap VirtualMetadataIdMap;
   MetadataTypeMap GeneralizedMetadataIdMap;
 
 public:
   CodeGenModule(ASTContext &C, const HeaderSearchOptions &headersearchopts,
                 const PreprocessorOptions &ppopts,
                 const CodeGenOptions &CodeGenOpts, llvm::Module &M,
                 DiagnosticsEngine &Diags,
                 CoverageSourceInfo *CoverageInfo = nullptr);
 
   ~CodeGenModule();
 
   void clear();
 
   /// Finalize LLVM code generation.
   void Release();
 
   /// Return true if we should emit location information for expressions.
   bool getExpressionLocationsEnabled() const;
 
   /// Return a reference to the configured Objective-C runtime.
   CGObjCRuntime &getObjCRuntime() {
     if (!ObjCRuntime) createObjCRuntime();
     return *ObjCRuntime;
   }
 
   /// Return true iff an Objective-C runtime has been configured.
   bool hasObjCRuntime() { return !!ObjCRuntime; }
 
   const std::string &getModuleNameHash() const { return ModuleNameHash; }
 
   /// Return a reference to the configured OpenCL runtime.
   CGOpenCLRuntime &getOpenCLRuntime() {
     assert(OpenCLRuntime != nullptr);
     return *OpenCLRuntime;
   }
 
   /// Return a reference to the configured OpenMP runtime.
   CGOpenMPRuntime &getOpenMPRuntime() {
     assert(OpenMPRuntime != nullptr);
     return *OpenMPRuntime;
   }
 
   /// Return a reference to the configured CUDA runtime.
   CGCUDARuntime &getCUDARuntime() {
     assert(CUDARuntime != nullptr);
     return *CUDARuntime;
   }
 
   ObjCEntrypoints &getObjCEntrypoints() const {
     assert(ObjCData != nullptr);
     return *ObjCData;
   }
 
   // Version checking functions, used to implement ObjC's @available:
   // i32 @__isOSVersionAtLeast(i32, i32, i32)
   llvm::FunctionCallee IsOSVersionAtLeastFn = nullptr;
   // i32 @__isPlatformVersionAtLeast(i32, i32, i32, i32)
   llvm::FunctionCallee IsPlatformVersionAtLeastFn = nullptr;
 
   InstrProfStats &getPGOStats() { return PGOStats; }
   llvm::IndexedInstrProfReader *getPGOReader() const { return PGOReader.get(); }
 
   CoverageMappingModuleGen *getCoverageMapping() const {
     return CoverageMapping.get();
   }
 
   llvm::Constant *getStaticLocalDeclAddress(const VarDecl *D) {
     return StaticLocalDeclMap[D];
   }
   void setStaticLocalDeclAddress(const VarDecl *D,
                                  llvm::Constant *C) {
     StaticLocalDeclMap[D] = C;
   }
 
   llvm::Constant *
   getOrCreateStaticVarDecl(const VarDecl &D,
                            llvm::GlobalValue::LinkageTypes Linkage);
 
   llvm::GlobalVariable *getStaticLocalDeclGuardAddress(const VarDecl *D) {
     return StaticLocalDeclGuardMap[D];
   }
   void setStaticLocalDeclGuardAddress(const VarDecl *D,
                                       llvm::GlobalVariable *C) {
     StaticLocalDeclGuardMap[D] = C;
   }
 
   Address createUnnamedGlobalFrom(const VarDecl &D, llvm::Constant *Constant,
                                   CharUnits Align);
 
   bool lookupRepresentativeDecl(StringRef MangledName,
                                 GlobalDecl &Result) const;
 
   llvm::Constant *getAtomicSetterHelperFnMap(QualType Ty) {
     return AtomicSetterHelperFnMap[Ty];
   }
   void setAtomicSetterHelperFnMap(QualType Ty,
                             llvm::Constant *Fn) {
     AtomicSetterHelperFnMap[Ty] = Fn;
   }
 
   llvm::Constant *getAtomicGetterHelperFnMap(QualType Ty) {
     return AtomicGetterHelperFnMap[Ty];
   }
   void setAtomicGetterHelperFnMap(QualType Ty,
                             llvm::Constant *Fn) {
     AtomicGetterHelperFnMap[Ty] = Fn;
   }
 
   llvm::Constant *getTypeDescriptorFromMap(QualType Ty) {
     return TypeDescriptorMap[Ty];
   }
   void setTypeDescriptorInMap(QualType Ty, llvm::Constant *C) {
     TypeDescriptorMap[Ty] = C;
   }
 
   CGDebugInfo *getModuleDebugInfo() { return DebugInfo.get(); }
 
   llvm::MDNode *getNoObjCARCExceptionsMetadata() {
     if (!NoObjCARCExceptionsMetadata)
       NoObjCARCExceptionsMetadata = llvm::MDNode::get(getLLVMContext(), None);
     return NoObjCARCExceptionsMetadata;
   }
 
   ASTContext &getContext() const { return Context; }
   const LangOptions &getLangOpts() const { return LangOpts; }
   const HeaderSearchOptions &getHeaderSearchOpts()
     const { return HeaderSearchOpts; }
   const PreprocessorOptions &getPreprocessorOpts()
     const { return PreprocessorOpts; }
   const CodeGenOptions &getCodeGenOpts() const { return CodeGenOpts; }
   llvm::Module &getModule() const { return TheModule; }
   DiagnosticsEngine &getDiags() const { return Diags; }
   const llvm::DataLayout &getDataLayout() const {
     return TheModule.getDataLayout();
   }
   const TargetInfo &getTarget() const { return Target; }
   const llvm::Triple &getTriple() const { return Target.getTriple(); }
   bool supportsCOMDAT() const;
   void maybeSetTrivialComdat(const Decl &D, llvm::GlobalObject &GO);
 
   CGCXXABI &getCXXABI() const { return *ABI; }
   llvm::LLVMContext &getLLVMContext() { return VMContext; }
 
   bool shouldUseTBAA() const { return TBAA != nullptr; }
 
   const TargetCodeGenInfo &getTargetCodeGenInfo();
 
   CodeGenTypes &getTypes() { return Types; }
 
   CodeGenVTables &getVTables() { return VTables; }
 
   ItaniumVTableContext &getItaniumVTableContext() {
     return VTables.getItaniumVTableContext();
   }
 
   MicrosoftVTableContext &getMicrosoftVTableContext() {
     return VTables.getMicrosoftVTableContext();
   }
 
   CtorList &getGlobalCtors() { return GlobalCtors; }
   CtorList &getGlobalDtors() { return GlobalDtors; }
 
   /// getTBAATypeInfo - Get metadata used to describe accesses to objects of
   /// the given type.
   llvm::MDNode *getTBAATypeInfo(QualType QTy);
 
   /// getTBAAAccessInfo - Get TBAA information that describes an access to
   /// an object of the given type.
   TBAAAccessInfo getTBAAAccessInfo(QualType AccessType);
 
   /// getTBAAVTablePtrAccessInfo - Get the TBAA information that describes an
   /// access to a virtual table pointer.
   TBAAAccessInfo getTBAAVTablePtrAccessInfo(llvm::Type *VTablePtrType);
 
   llvm::MDNode *getTBAAStructInfo(QualType QTy);
 
   /// getTBAABaseTypeInfo - Get metadata that describes the given base access
   /// type. Return null if the type is not suitable for use in TBAA access tags.
   llvm::MDNode *getTBAABaseTypeInfo(QualType QTy);
 
   /// getTBAAAccessTagInfo - Get TBAA tag for a given memory access.
   llvm::MDNode *getTBAAAccessTagInfo(TBAAAccessInfo Info);
 
   /// mergeTBAAInfoForCast - Get merged TBAA information for the purposes of
   /// type casts.
   TBAAAccessInfo mergeTBAAInfoForCast(TBAAAccessInfo SourceInfo,
                                       TBAAAccessInfo TargetInfo);
 
   /// mergeTBAAInfoForConditionalOperator - Get merged TBAA information for the
   /// purposes of conditional operator.
   TBAAAccessInfo mergeTBAAInfoForConditionalOperator(TBAAAccessInfo InfoA,
                                                      TBAAAccessInfo InfoB);
 
   /// mergeTBAAInfoForMemoryTransfer - Get merged TBAA information for the
   /// purposes of memory transfer calls.
   TBAAAccessInfo mergeTBAAInfoForMemoryTransfer(TBAAAccessInfo DestInfo,
                                                 TBAAAccessInfo SrcInfo);
 
   /// getTBAAInfoForSubobject - Get TBAA information for an access with a given
   /// base lvalue.
   TBAAAccessInfo getTBAAInfoForSubobject(LValue Base, QualType AccessType) {
     if (Base.getTBAAInfo().isMayAlias())
       return TBAAAccessInfo::getMayAliasInfo();
     return getTBAAAccessInfo(AccessType);
   }
 
   bool isTypeConstant(QualType QTy, bool ExcludeCtorDtor);
 
   bool isPaddedAtomicType(QualType type);
   bool isPaddedAtomicType(const AtomicType *type);
 
   /// DecorateInstructionWithTBAA - Decorate the instruction with a TBAA tag.
   void DecorateInstructionWithTBAA(llvm::Instruction *Inst,
                                    TBAAAccessInfo TBAAInfo);
 
   /// Adds !invariant.barrier !tag to instruction
   void DecorateInstructionWithInvariantGroup(llvm::Instruction *I,
                                              const CXXRecordDecl *RD);
 
   /// Emit the given number of characters as a value of type size_t.
   llvm::ConstantInt *getSize(CharUnits numChars);
 
   /// Set the visibility for the given LLVM GlobalValue.
   void setGlobalVisibility(llvm::GlobalValue *GV, const NamedDecl *D) const;
 
   void setDSOLocal(llvm::GlobalValue *GV) const;
 
   void setDLLImportDLLExport(llvm::GlobalValue *GV, GlobalDecl D) const;
   void setDLLImportDLLExport(llvm::GlobalValue *GV, const NamedDecl *D) const;
   /// Set visibility, dllimport/dllexport and dso_local.
   /// This must be called after dllimport/dllexport is set.
   void setGVProperties(llvm::GlobalValue *GV, GlobalDecl GD) const;
   void setGVProperties(llvm::GlobalValue *GV, const NamedDecl *D) const;
 
   void setGVPropertiesAux(llvm::GlobalValue *GV, const NamedDecl *D) const;
 
   /// Set the TLS mode for the given LLVM GlobalValue for the thread-local
   /// variable declaration D.
   void setTLSMode(llvm::GlobalValue *GV, const VarDecl &D) const;
 
   /// Get LLVM TLS mode from CodeGenOptions.
   llvm::GlobalVariable::ThreadLocalMode GetDefaultLLVMTLSModel() const;
 
   static llvm::GlobalValue::VisibilityTypes GetLLVMVisibility(Visibility V) {
     switch (V) {
     case DefaultVisibility:   return llvm::GlobalValue::DefaultVisibility;
     case HiddenVisibility:    return llvm::GlobalValue::HiddenVisibility;
     case ProtectedVisibility: return llvm::GlobalValue::ProtectedVisibility;
     }
     llvm_unreachable("unknown visibility!");
   }
 
   llvm::Constant *GetAddrOfGlobal(GlobalDecl GD,
                                   ForDefinition_t IsForDefinition
                                     = NotForDefinition);
 
   /// Will return a global variable of the given type. If a variable with a
   /// different type already exists then a new  variable with the right type
   /// will be created and all uses of the old variable will be replaced with a
   /// bitcast to the new variable.
   llvm::GlobalVariable *
   CreateOrReplaceCXXRuntimeVariable(StringRef Name, llvm::Type *Ty,
                                     llvm::GlobalValue::LinkageTypes Linkage,
                                     unsigned Alignment);
 
   llvm::Function *CreateGlobalInitOrCleanUpFunction(
       llvm::FunctionType *ty, const Twine &name, const CGFunctionInfo &FI,
       SourceLocation Loc = SourceLocation(), bool TLS = false);
 
   /// Return the AST address space of the underlying global variable for D, as
   /// determined by its declaration. Normally this is the same as the address
   /// space of D's type, but in CUDA, address spaces are associated with
   /// declarations, not types. If D is nullptr, return the default address
   /// space for global variable.
   ///
   /// For languages without explicit address spaces, if D has default address
   /// space, target-specific global or constant address space may be returned.
   LangAS GetGlobalVarAddressSpace(const VarDecl *D);
 
   /// Return the AST address space of constant literal, which is used to emit
   /// the constant literal as global variable in LLVM IR.
   /// Note: This is not necessarily the address space of the constant literal
   /// in AST. For address space agnostic language, e.g. C++, constant literal
   /// in AST is always in default address space.
   LangAS GetGlobalConstantAddressSpace() const;
 
   /// Return the llvm::Constant for the address of the given global variable.
   /// If Ty is non-null and if the global doesn't exist, then it will be created
   /// with the specified type instead of whatever the normal requested type
   /// would be. If IsForDefinition is true, it is guaranteed that an actual
   /// global with type Ty will be returned, not conversion of a variable with
   /// the same mangled name but some other type.
   llvm::Constant *GetAddrOfGlobalVar(const VarDecl *D,
                                      llvm::Type *Ty = nullptr,
                                      ForDefinition_t IsForDefinition
                                        = NotForDefinition);
 
   /// Return the address of the given function. If Ty is non-null, then this
   /// function will use the specified type if it has to create it.
   llvm::Constant *GetAddrOfFunction(GlobalDecl GD, llvm::Type *Ty = nullptr,
                                     bool ForVTable = false,
                                     bool DontDefer = false,
                                     ForDefinition_t IsForDefinition
                                       = NotForDefinition);
 
   // Return the function body address of the given function.
   llvm::Constant *GetFunctionStart(const ValueDecl *Decl);
 
   /// Get the address of the RTTI descriptor for the given type.
   llvm::Constant *GetAddrOfRTTIDescriptor(QualType Ty, bool ForEH = false);
 
   /// Get the address of a GUID.
   ConstantAddress GetAddrOfMSGuidDecl(const MSGuidDecl *GD);
 
   /// Get the address of a template parameter object.
   ConstantAddress
   GetAddrOfTemplateParamObject(const TemplateParamObjectDecl *TPO);
 
   /// Get the address of the thunk for the given global decl.
   llvm::Constant *GetAddrOfThunk(StringRef Name, llvm::Type *FnTy,
                                  GlobalDecl GD);
 
   /// Get a reference to the target of VD.
   ConstantAddress GetWeakRefReference(const ValueDecl *VD);
 
   /// Returns the assumed alignment of an opaque pointer to the given class.
   CharUnits getClassPointerAlignment(const CXXRecordDecl *CD);
 
   /// Returns the minimum object size for an object of the given class type
   /// (or a class derived from it).
   CharUnits getMinimumClassObjectSize(const CXXRecordDecl *CD);
 
   /// Returns the minimum object size for an object of the given type.
   CharUnits getMinimumObjectSize(QualType Ty) {
     if (CXXRecordDecl *RD = Ty->getAsCXXRecordDecl())
       return getMinimumClassObjectSize(RD);
     return getContext().getTypeSizeInChars(Ty);
   }
 
   /// Returns the assumed alignment of a virtual base of a class.
   CharUnits getVBaseAlignment(CharUnits DerivedAlign,
                               const CXXRecordDecl *Derived,
                               const CXXRecordDecl *VBase);
 
   /// Given a class pointer with an actual known alignment, and the
   /// expected alignment of an object at a dynamic offset w.r.t that
   /// pointer, return the alignment to assume at the offset.
   CharUnits getDynamicOffsetAlignment(CharUnits ActualAlign,
                                       const CXXRecordDecl *Class,
                                       CharUnits ExpectedTargetAlign);
 
   CharUnits
   computeNonVirtualBaseClassOffset(const CXXRecordDecl *DerivedClass,
                                    CastExpr::path_const_iterator Start,
                                    CastExpr::path_const_iterator End);
 
   /// Returns the offset from a derived class to  a class. Returns null if the
   /// offset is 0.
   llvm::Constant *
   GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
                                CastExpr::path_const_iterator PathBegin,
                                CastExpr::path_const_iterator PathEnd);
 
   llvm::FoldingSet<BlockByrefHelpers> ByrefHelpersCache;
 
   /// Fetches the global unique block count.
   int getUniqueBlockCount() { return ++Block.GlobalUniqueCount; }
 
   /// Fetches the type of a generic block descriptor.
   llvm::Type *getBlockDescriptorType();
 
   /// The type of a generic block literal.
   llvm::Type *getGenericBlockLiteralType();
 
   /// Gets the address of a block which requires no captures.
   llvm::Constant *GetAddrOfGlobalBlock(const BlockExpr *BE, StringRef Name);
 
   /// Returns the address of a block which requires no caputres, or null if
   /// we've yet to emit the block for BE.
   llvm::Constant *getAddrOfGlobalBlockIfEmitted(const BlockExpr *BE) {
     return EmittedGlobalBlocks.lookup(BE);
   }
 
   /// Notes that BE's global block is available via Addr. Asserts that BE
   /// isn't already emitted.
   void setAddrOfGlobalBlock(const BlockExpr *BE, llvm::Constant *Addr);
 
   /// Return a pointer to a constant CFString object for the given string.
   ConstantAddress GetAddrOfConstantCFString(const StringLiteral *Literal);
 
   /// Return a pointer to a constant NSString object for the given string. Or a
   /// user defined String object as defined via
   /// -fconstant-string-class=class_name option.
   ConstantAddress GetAddrOfConstantString(const StringLiteral *Literal);
 
   /// Return a constant array for the given string.
   llvm::Constant *GetConstantArrayFromStringLiteral(const StringLiteral *E);
 
   /// Return a pointer to a constant array for the given string literal.
   ConstantAddress
   GetAddrOfConstantStringFromLiteral(const StringLiteral *S,
                                      StringRef Name = ".str");
 
   /// Return a pointer to a constant array for the given ObjCEncodeExpr node.
   ConstantAddress
   GetAddrOfConstantStringFromObjCEncode(const ObjCEncodeExpr *);
 
   /// Returns a pointer to a character array containing the literal and a
   /// terminating '\0' character. The result has pointer to array type.
   ///
   /// \param GlobalName If provided, the name to use for the global (if one is
   /// created).
   ConstantAddress
   GetAddrOfConstantCString(const std::string &Str,
                            const char *GlobalName = nullptr);
 
   /// Returns a pointer to a constant global variable for the given file-scope
   /// compound literal expression.
   ConstantAddress GetAddrOfConstantCompoundLiteral(const CompoundLiteralExpr*E);
 
   /// If it's been emitted already, returns the GlobalVariable corresponding to
   /// a compound literal. Otherwise, returns null.
   llvm::GlobalVariable *
   getAddrOfConstantCompoundLiteralIfEmitted(const CompoundLiteralExpr *E);
 
   /// Notes that CLE's GlobalVariable is GV. Asserts that CLE isn't already
   /// emitted.
   void setAddrOfConstantCompoundLiteral(const CompoundLiteralExpr *CLE,
                                         llvm::GlobalVariable *GV);
 
   /// Returns a pointer to a global variable representing a temporary
   /// with static or thread storage duration.
   ConstantAddress GetAddrOfGlobalTemporary(const MaterializeTemporaryExpr *E,
                                            const Expr *Inner);
 
   /// Retrieve the record type that describes the state of an
   /// Objective-C fast enumeration loop (for..in).
   QualType getObjCFastEnumerationStateType();
 
   // Produce code for this constructor/destructor. This method doesn't try
   // to apply any ABI rules about which other constructors/destructors
   // are needed or if they are alias to each other.
   llvm::Function *codegenCXXStructor(GlobalDecl GD);
 
   /// Return the address of the constructor/destructor of the given type.
   llvm::Constant *
   getAddrOfCXXStructor(GlobalDecl GD, const CGFunctionInfo *FnInfo = nullptr,
                        llvm::FunctionType *FnType = nullptr,
                        bool DontDefer = false,
                        ForDefinition_t IsForDefinition = NotForDefinition) {
     return cast<llvm::Constant>(getAddrAndTypeOfCXXStructor(GD, FnInfo, FnType,
                                                             DontDefer,
                                                             IsForDefinition)
                                     .getCallee());
   }
 
   llvm::FunctionCallee getAddrAndTypeOfCXXStructor(
       GlobalDecl GD, const CGFunctionInfo *FnInfo = nullptr,
       llvm::FunctionType *FnType = nullptr, bool DontDefer = false,
       ForDefinition_t IsForDefinition = NotForDefinition);
 
   /// Given a builtin id for a function like "__builtin_fabsf", return a
   /// Function* for "fabsf".
   llvm::Constant *getBuiltinLibFunction(const FunctionDecl *FD,
                                         unsigned BuiltinID);
 
   llvm::Function *getIntrinsic(unsigned IID, ArrayRef<llvm::Type*> Tys = None);
 
   /// Emit code for a single top level declaration.
   void EmitTopLevelDecl(Decl *D);
 
   /// Stored a deferred empty coverage mapping for an unused
   /// and thus uninstrumented top level declaration.
   void AddDeferredUnusedCoverageMapping(Decl *D);
 
   /// Remove the deferred empty coverage mapping as this
   /// declaration is actually instrumented.
   void ClearUnusedCoverageMapping(const Decl *D);
 
   /// Emit all the deferred coverage mappings
   /// for the uninstrumented functions.
   void EmitDeferredUnusedCoverageMappings();
 
   /// Emit an alias for "main" if it has no arguments (needed for wasm).
   void EmitMainVoidAlias();
 
   /// Tell the consumer that this variable has been instantiated.
   void HandleCXXStaticMemberVarInstantiation(VarDecl *VD);
 
   /// If the declaration has internal linkage but is inside an
   /// extern "C" linkage specification, prepare to emit an alias for it
   /// to the expected name.
   template<typename SomeDecl>
   void MaybeHandleStaticInExternC(const SomeDecl *D, llvm::GlobalValue *GV);
 
   /// Add a global to a list to be added to the llvm.used metadata.
   void addUsedGlobal(llvm::GlobalValue *GV);
 
   /// Add a global to a list to be added to the llvm.compiler.used metadata.
   void addCompilerUsedGlobal(llvm::GlobalValue *GV);
 
   /// Add a global to a list to be added to the llvm.compiler.used metadata.
   void addUsedOrCompilerUsedGlobal(llvm::GlobalValue *GV);
 
   /// Add a destructor and object to add to the C++ global destructor function.
   void AddCXXDtorEntry(llvm::FunctionCallee DtorFn, llvm::Constant *Object) {
     CXXGlobalDtorsOrStermFinalizers.emplace_back(DtorFn.getFunctionType(),
                                                  DtorFn.getCallee(), Object);
   }
 
   /// Add an sterm finalizer to the C++ global cleanup function.
   void AddCXXStermFinalizerEntry(llvm::FunctionCallee DtorFn) {
     CXXGlobalDtorsOrStermFinalizers.emplace_back(DtorFn.getFunctionType(),
                                                  DtorFn.getCallee(), nullptr);
   }
 
   /// Add an sterm finalizer to its own llvm.global_dtors entry.
   void AddCXXStermFinalizerToGlobalDtor(llvm::Function *StermFinalizer,
                                         int Priority) {
     AddGlobalDtor(StermFinalizer, Priority);
   }
 
   void AddCXXPrioritizedStermFinalizerEntry(llvm::Function *StermFinalizer,
                                             int Priority) {
     OrderGlobalInitsOrStermFinalizers Key(Priority,
                                           PrioritizedCXXStermFinalizers.size());
     PrioritizedCXXStermFinalizers.push_back(
         std::make_pair(Key, StermFinalizer));
   }
 
   /// Create or return a runtime function declaration with the specified type
   /// and name. If \p AssumeConvergent is true, the call will have the
   /// convergent attribute added.
   llvm::FunctionCallee
   CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name,
                         llvm::AttributeList ExtraAttrs = llvm::AttributeList(),
                         bool Local = false, bool AssumeConvergent = false);
 
   /// Create a new runtime global variable with the specified type and name.
   llvm::Constant *CreateRuntimeVariable(llvm::Type *Ty,
                                         StringRef Name);
 
   ///@name Custom Blocks Runtime Interfaces
   ///@{
 
   llvm::Constant *getNSConcreteGlobalBlock();
   llvm::Constant *getNSConcreteStackBlock();
   llvm::FunctionCallee getBlockObjectAssign();
   llvm::FunctionCallee getBlockObjectDispose();
 
   ///@}
 
   llvm::Function *getLLVMLifetimeStartFn();
   llvm::Function *getLLVMLifetimeEndFn();
 
   // Make sure that this type is translated.
   void UpdateCompletedType(const TagDecl *TD);
 
   llvm::Constant *getMemberPointerConstant(const UnaryOperator *e);
 
   /// Emit type info if type of an expression is a variably modified
   /// type. Also emit proper debug info for cast types.
   void EmitExplicitCastExprType(const ExplicitCastExpr *E,
                                 CodeGenFunction *CGF = nullptr);
 
   /// Return the result of value-initializing the given type, i.e. a null
   /// expression of the given type.  This is usually, but not always, an LLVM
   /// null constant.
   llvm::Constant *EmitNullConstant(QualType T);
 
   /// Return a null constant appropriate for zero-initializing a base class with
   /// the given type. This is usually, but not always, an LLVM null constant.
   llvm::Constant *EmitNullConstantForBase(const CXXRecordDecl *Record);
 
   /// Emit a general error that something can't be done.
   void Error(SourceLocation loc, StringRef error);
 
   /// Print out an error that codegen doesn't support the specified stmt yet.
   void ErrorUnsupported(const Stmt *S, const char *Type);
 
   /// Print out an error that codegen doesn't support the specified decl yet.
   void ErrorUnsupported(const Decl *D, const char *Type);
 
   /// Set the attributes on the LLVM function for the given decl and function
   /// info. This applies attributes necessary for handling the ABI as well as
   /// user specified attributes like section.
   void SetInternalFunctionAttributes(GlobalDecl GD, llvm::Function *F,
                                      const CGFunctionInfo &FI);
 
   /// Set the LLVM function attributes (sext, zext, etc).
   void SetLLVMFunctionAttributes(GlobalDecl GD, const CGFunctionInfo &Info,
                                  llvm::Function *F, bool IsThunk);
 
   /// Set the LLVM function attributes which only apply to a function
   /// definition.
   void SetLLVMFunctionAttributesForDefinition(const Decl *D, llvm::Function *F);
 
   /// Set the LLVM function attributes that represent floating point
   /// environment.
   void setLLVMFunctionFEnvAttributes(const FunctionDecl *D, llvm::Function *F);
 
   /// Return true iff the given type uses 'sret' when used as a return type.
   bool ReturnTypeUsesSRet(const CGFunctionInfo &FI);
 
   /// Return true iff the given type uses an argument slot when 'sret' is used
   /// as a return type.
   bool ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI);
 
   /// Return true iff the given type uses 'fpret' when used as a return type.
   bool ReturnTypeUsesFPRet(QualType ResultType);
 
   /// Return true iff the given type uses 'fp2ret' when used as a return type.
   bool ReturnTypeUsesFP2Ret(QualType ResultType);
 
   /// Get the LLVM attributes and calling convention to use for a particular
   /// function type.
   ///
   /// \param Name - The function name.
   /// \param Info - The function type information.
   /// \param CalleeInfo - The callee information these attributes are being
   /// constructed for. If valid, the attributes applied to this decl may
   /// contribute to the function attributes and calling convention.
   /// \param Attrs [out] - On return, the attribute list to use.
   /// \param CallingConv [out] - On return, the LLVM calling convention to use.
   void ConstructAttributeList(StringRef Name, const CGFunctionInfo &Info,
                               CGCalleeInfo CalleeInfo,
                               llvm::AttributeList &Attrs, unsigned &CallingConv,
                               bool AttrOnCallSite, bool IsThunk);
 
   /// Adds attributes to F according to our CodeGenOptions and LangOptions, as
   /// though we had emitted it ourselves.  We remove any attributes on F that
   /// conflict with the attributes we add here.
   ///
   /// This is useful for adding attrs to bitcode modules that you want to link
   /// with but don't control, such as CUDA's libdevice.  When linking with such
   /// a bitcode library, you might want to set e.g. its functions'
   /// "unsafe-fp-math" attribute to match the attr of the functions you're
   /// codegen'ing.  Otherwise, LLVM will interpret the bitcode module's lack of
   /// unsafe-fp-math attrs as tantamount to unsafe-fp-math=false, and then LLVM
   /// will propagate unsafe-fp-math=false up to every transitive caller of a
   /// function in the bitcode library!
   ///
   /// With the exception of fast-math attrs, this will only make the attributes
   /// on the function more conservative.  But it's unsafe to call this on a
   /// function which relies on particular fast-math attributes for correctness.
   /// It's up to you to ensure that this is safe.
   void addDefaultFunctionDefinitionAttributes(llvm::Function &F);
 
   /// Like the overload taking a `Function &`, but intended specifically
   /// for frontends that want to build on Clang's target-configuration logic.
   void addDefaultFunctionDefinitionAttributes(llvm::AttrBuilder &attrs);
 
   StringRef getMangledName(GlobalDecl GD);
   StringRef getBlockMangledName(GlobalDecl GD, const BlockDecl *BD);
 
   void EmitTentativeDefinition(const VarDecl *D);
 
   void EmitExternalDeclaration(const VarDecl *D);
 
   void EmitVTable(CXXRecordDecl *Class);
 
   void RefreshTypeCacheForClass(const CXXRecordDecl *Class);
 
   /// Appends Opts to the "llvm.linker.options" metadata value.
   void AppendLinkerOptions(StringRef Opts);
 
   /// Appends a detect mismatch command to the linker options.
   void AddDetectMismatch(StringRef Name, StringRef Value);
 
   /// Appends a dependent lib to the appropriate metadata value.
   void AddDependentLib(StringRef Lib);
 
 
   llvm::GlobalVariable::LinkageTypes getFunctionLinkage(GlobalDecl GD);
 
   void setFunctionLinkage(GlobalDecl GD, llvm::Function *F) {
     F->setLinkage(getFunctionLinkage(GD));
   }
 
   /// Return the appropriate linkage for the vtable, VTT, and type information
   /// of the given class.
   llvm::GlobalVariable::LinkageTypes getVTableLinkage(const CXXRecordDecl *RD);
 
   /// Return the store size, in character units, of the given LLVM type.
   CharUnits GetTargetTypeStoreSize(llvm::Type *Ty) const;
 
   /// Returns LLVM linkage for a declarator.
   llvm::GlobalValue::LinkageTypes
   getLLVMLinkageForDeclarator(const DeclaratorDecl *D, GVALinkage Linkage,
                               bool IsConstantVariable);
 
   /// Returns LLVM linkage for a declarator.
   llvm::GlobalValue::LinkageTypes
   getLLVMLinkageVarDefinition(const VarDecl *VD, bool IsConstant);
 
   /// Emit all the global annotations.
   void EmitGlobalAnnotations();
 
   /// Emit an annotation string.
   llvm::Constant *EmitAnnotationString(StringRef Str);
 
   /// Emit the annotation's translation unit.
   llvm::Constant *EmitAnnotationUnit(SourceLocation Loc);
 
   /// Emit the annotation line number.
   llvm::Constant *EmitAnnotationLineNo(SourceLocation L);
 
   /// Emit additional args of the annotation.
   llvm::Constant *EmitAnnotationArgs(const AnnotateAttr *Attr);
 
   /// Generate the llvm::ConstantStruct which contains the annotation
   /// information for a given GlobalValue. The annotation struct is
   /// {i8 *, i8 *, i8 *, i32}. The first field is a constant expression, the
   /// GlobalValue being annotated. The second field is the constant string
   /// created from the AnnotateAttr's annotation. The third field is a constant
   /// string containing the name of the translation unit. The fourth field is
   /// the line number in the file of the annotated value declaration.
   llvm::Constant *EmitAnnotateAttr(llvm::GlobalValue *GV,
                                    const AnnotateAttr *AA,
                                    SourceLocation L);
 
   /// Add global annotations that are set on D, for the global GV. Those
   /// annotations are emitted during finalization of the LLVM code.
   void AddGlobalAnnotations(const ValueDecl *D, llvm::GlobalValue *GV);
 
   bool isInNoSanitizeList(SanitizerMask Kind, llvm::Function *Fn,
                           SourceLocation Loc) const;
 
   bool isInNoSanitizeList(llvm::GlobalVariable *GV, SourceLocation Loc,
                           QualType Ty, StringRef Category = StringRef()) const;
 
   /// Imbue XRay attributes to a function, applying the always/never attribute
   /// lists in the process. Returns true if we did imbue attributes this way,
   /// false otherwise.
   bool imbueXRayAttrs(llvm::Function *Fn, SourceLocation Loc,
                       StringRef Category = StringRef()) const;
 
   /// Returns true if function at the given location should be excluded from
   /// profile instrumentation.
   bool isProfileInstrExcluded(llvm::Function *Fn, SourceLocation Loc) const;
 
   SanitizerMetadata *getSanitizerMetadata() {
     return SanitizerMD.get();
   }
 
   void addDeferredVTable(const CXXRecordDecl *RD) {
     DeferredVTables.push_back(RD);
   }
 
   /// Emit code for a single global function or var decl. Forward declarations
   /// are emitted lazily.
   void EmitGlobal(GlobalDecl D);
 
   bool TryEmitBaseDestructorAsAlias(const CXXDestructorDecl *D);
 
   llvm::GlobalValue *GetGlobalValue(StringRef Ref);
 
   /// Set attributes which are common to any form of a global definition (alias,
   /// Objective-C method, function, global variable).
   ///
   /// NOTE: This should only be called for definitions.
   void SetCommonAttributes(GlobalDecl GD, llvm::GlobalValue *GV);
 
   void addReplacement(StringRef Name, llvm::Constant *C);
 
   void addGlobalValReplacement(llvm::GlobalValue *GV, llvm::Constant *C);
 
   /// Emit a code for threadprivate directive.
   /// \param D Threadprivate declaration.
   void EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D);
 
   /// Emit a code for declare reduction construct.
   void EmitOMPDeclareReduction(const OMPDeclareReductionDecl *D,
                                CodeGenFunction *CGF = nullptr);
 
   /// Emit a code for declare mapper construct.
   void EmitOMPDeclareMapper(const OMPDeclareMapperDecl *D,
                             CodeGenFunction *CGF = nullptr);
 
   /// Emit a code for requires directive.
   /// \param D Requires declaration
   void EmitOMPRequiresDecl(const OMPRequiresDecl *D);
 
   /// Emit a code for the allocate directive.
   /// \param D The allocate declaration
   void EmitOMPAllocateDecl(const OMPAllocateDecl *D);
 
   /// Returns whether the given record has hidden LTO visibility and therefore
   /// may participate in (single-module) CFI and whole-program vtable
   /// optimization.
   bool HasHiddenLTOVisibility(const CXXRecordDecl *RD);
 
   /// Returns whether the given record has public std LTO visibility
   /// and therefore may not participate in (single-module) CFI and whole-program
   /// vtable optimization.
   bool HasLTOVisibilityPublicStd(const CXXRecordDecl *RD);
 
   /// Returns the vcall visibility of the given type. This is the scope in which
   /// a virtual function call could be made which ends up being dispatched to a
   /// member function of this class. This scope can be wider than the visibility
   /// of the class itself when the class has a more-visible dynamic base class.
   /// The client should pass in an empty Visited set, which is used to prevent
   /// redundant recursive processing.
   llvm::GlobalObject::VCallVisibility
   GetVCallVisibilityLevel(const CXXRecordDecl *RD,
                           llvm::DenseSet<const CXXRecordDecl *> &Visited);
 
   /// Emit type metadata for the given vtable using the given layout.
   void EmitVTableTypeMetadata(const CXXRecordDecl *RD,
                               llvm::GlobalVariable *VTable,
                               const VTableLayout &VTLayout);
 
   /// Generate a cross-DSO type identifier for MD.
   llvm::ConstantInt *CreateCrossDsoCfiTypeId(llvm::Metadata *MD);
 
   /// Create a metadata identifier for the given type. This may either be an
   /// MDString (for external identifiers) or a distinct unnamed MDNode (for
   /// internal identifiers).
   llvm::Metadata *CreateMetadataIdentifierForType(QualType T);
 
   /// Create a metadata identifier that is intended to be used to check virtual
   /// calls via a member function pointer.
   llvm::Metadata *CreateMetadataIdentifierForVirtualMemPtrType(QualType T);
 
   /// Create a metadata identifier for the generalization of the given type.
   /// This may either be an MDString (for external identifiers) or a distinct
   /// unnamed MDNode (for internal identifiers).
   llvm::Metadata *CreateMetadataIdentifierGeneralized(QualType T);
 
   /// Create and attach type metadata to the given function.
   void CreateFunctionTypeMetadataForIcall(const FunctionDecl *FD,
                                           llvm::Function *F);
 
   /// Whether this function's return type has no side effects, and thus may
   /// be trivially discarded if it is unused.
   bool MayDropFunctionReturn(const ASTContext &Context, QualType ReturnType);
 
   /// Returns whether this module needs the "all-vtables" type identifier.
   bool NeedAllVtablesTypeId() const;
 
   /// Create and attach type metadata for the given vtable.
   void AddVTableTypeMetadata(llvm::GlobalVariable *VTable, CharUnits Offset,
                              const CXXRecordDecl *RD);
 
   /// Return a vector of most-base classes for RD. This is used to implement
   /// control flow integrity checks for member function pointers.
   ///
   /// A most-base class of a class C is defined as a recursive base class of C,
   /// including C itself, that does not have any bases.
   std::vector<const CXXRecordDecl *>
   getMostBaseClasses(const CXXRecordDecl *RD);
 
   /// Get the declaration of std::terminate for the platform.
   llvm::FunctionCallee getTerminateFn();
 
   llvm::SanitizerStatReport &getSanStats();
 
   llvm::Value *
   createOpenCLIntToSamplerConversion(const Expr *E, CodeGenFunction &CGF);
 
   /// OpenCL v1.2 s5.6.4.6 allows the compiler to store kernel argument
   /// information in the program executable. The argument information stored
   /// includes the argument name, its type, the address and access qualifiers
   /// used. This helper can be used to generate metadata for source code kernel
   /// function as well as generated implicitly kernels. If a kernel is generated
   /// implicitly null value has to be passed to the last two parameters,
   /// otherwise all parameters must have valid non-null values.
   /// \param FN is a pointer to IR function being generated.
   /// \param FD is a pointer to function declaration if any.
   /// \param CGF is a pointer to CodeGenFunction that generates this function.
   void GenOpenCLArgMetadata(llvm::Function *FN,
                             const FunctionDecl *FD = nullptr,
                             CodeGenFunction *CGF = nullptr);
 
   /// Get target specific null pointer.
   /// \param T is the LLVM type of the null pointer.
   /// \param QT is the clang QualType of the null pointer.
   llvm::Constant *getNullPointer(llvm::PointerType *T, QualType QT);
 
   CharUnits getNaturalTypeAlignment(QualType T,
                                     LValueBaseInfo *BaseInfo = nullptr,
                                     TBAAAccessInfo *TBAAInfo = nullptr,
                                     bool forPointeeType = false);
   CharUnits getNaturalPointeeTypeAlignment(QualType T,
                                            LValueBaseInfo *BaseInfo = nullptr,
                                            TBAAAccessInfo *TBAAInfo = nullptr);
   bool stopAutoInit();
 
-  /// Print the postfix for externalized static variable for single source
-  /// offloading languages CUDA and HIP.
-  void printPostfixForExternalizedStaticVar(llvm::raw_ostream &OS) const;
+  /// Print the postfix for externalized static variable or kernels for single
+  /// source offloading languages CUDA and HIP.
+  void printPostfixForExternalizedDecl(llvm::raw_ostream &OS,
+                                       const Decl *D) const;
 
 private:
   llvm::Constant *GetOrCreateLLVMFunction(
       StringRef MangledName, llvm::Type *Ty, GlobalDecl D, bool ForVTable,
       bool DontDefer = false, bool IsThunk = false,
       llvm::AttributeList ExtraAttrs = llvm::AttributeList(),
       ForDefinition_t IsForDefinition = NotForDefinition);
 
   llvm::Constant *GetOrCreateMultiVersionResolver(GlobalDecl GD,
                                                   llvm::Type *DeclTy,
                                                   const FunctionDecl *FD);
   void UpdateMultiVersionNames(GlobalDecl GD, const FunctionDecl *FD,
                                StringRef &CurName);
 
   llvm::Constant *
   GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty, LangAS AddrSpace,
                         const VarDecl *D,
                         ForDefinition_t IsForDefinition = NotForDefinition);
 
   bool GetCPUAndFeaturesAttributes(GlobalDecl GD,
                                    llvm::AttrBuilder &AttrBuilder);
   void setNonAliasAttributes(GlobalDecl GD, llvm::GlobalObject *GO);
 
   /// Set function attributes for a function declaration.
   void SetFunctionAttributes(GlobalDecl GD, llvm::Function *F,
                              bool IsIncompleteFunction, bool IsThunk);
 
   void EmitGlobalDefinition(GlobalDecl D, llvm::GlobalValue *GV = nullptr);
 
   void EmitGlobalFunctionDefinition(GlobalDecl GD, llvm::GlobalValue *GV);
   void EmitMultiVersionFunctionDefinition(GlobalDecl GD, llvm::GlobalValue *GV);
 
   void EmitGlobalVarDefinition(const VarDecl *D, bool IsTentative = false);
   void EmitExternalVarDeclaration(const VarDecl *D);
   void EmitAliasDefinition(GlobalDecl GD);
   void emitIFuncDefinition(GlobalDecl GD);
   void emitCPUDispatchDefinition(GlobalDecl GD);
   void EmitTargetClonesResolver(GlobalDecl GD);
   void EmitObjCPropertyImplementations(const ObjCImplementationDecl *D);
   void EmitObjCIvarInitializations(ObjCImplementationDecl *D);
 
   // C++ related functions.
 
   void EmitDeclContext(const DeclContext *DC);
   void EmitLinkageSpec(const LinkageSpecDecl *D);
 
   /// Emit the function that initializes C++ thread_local variables.
   void EmitCXXThreadLocalInitFunc();
 
   /// Emit the function that initializes C++ globals.
   void EmitCXXGlobalInitFunc();
 
   /// Emit the function that performs cleanup associated with C++ globals.
   void EmitCXXGlobalCleanUpFunc();
 
   /// Emit the function that initializes the specified global (if PerformInit is
   /// true) and registers its destructor.
   void EmitCXXGlobalVarDeclInitFunc(const VarDecl *D,
                                     llvm::GlobalVariable *Addr,
                                     bool PerformInit);
 
   void EmitPointerToInitFunc(const VarDecl *VD, llvm::GlobalVariable *Addr,
                              llvm::Function *InitFunc, InitSegAttr *ISA);
 
   // FIXME: Hardcoding priority here is gross.
   void AddGlobalCtor(llvm::Function *Ctor, int Priority = 65535,
                      llvm::Constant *AssociatedData = nullptr);
   void AddGlobalDtor(llvm::Function *Dtor, int Priority = 65535,
                      bool IsDtorAttrFunc = false);
 
   /// EmitCtorList - Generates a global array of functions and priorities using
   /// the given list and name. This array will have appending linkage and is
   /// suitable for use as a LLVM constructor or destructor array. Clears Fns.
   void EmitCtorList(CtorList &Fns, const char *GlobalName);
 
   /// Emit any needed decls for which code generation was deferred.
   void EmitDeferred();
 
   /// Try to emit external vtables as available_externally if they have emitted
   /// all inlined virtual functions.  It runs after EmitDeferred() and therefore
   /// is not allowed to create new references to things that need to be emitted
   /// lazily.
   void EmitVTablesOpportunistically();
 
   /// Call replaceAllUsesWith on all pairs in Replacements.
   void applyReplacements();
 
   /// Call replaceAllUsesWith on all pairs in GlobalValReplacements.
   void applyGlobalValReplacements();
 
   void checkAliases();
 
   std::map<int, llvm::TinyPtrVector<llvm::Function *>> DtorsUsingAtExit;
 
   /// Register functions annotated with __attribute__((destructor)) using
   /// __cxa_atexit, if it is available, or atexit otherwise.
   void registerGlobalDtorsWithAtExit();
 
   // When using sinit and sterm functions, unregister
   // __attribute__((destructor)) annotated functions which were previously
   // registered by the atexit subroutine using unatexit.
   void unregisterGlobalDtorsWithUnAtExit();
 
   void emitMultiVersionFunctions();
 
   /// Emit any vtables which we deferred and still have a use for.
   void EmitDeferredVTables();
 
   /// Emit a dummy function that reference a CoreFoundation symbol when
   /// @available is used on Darwin.
   void emitAtAvailableLinkGuard();
 
   /// Emit the llvm.used and llvm.compiler.used metadata.
   void emitLLVMUsed();
 
   /// Emit the link options introduced by imported modules.
   void EmitModuleLinkOptions();
 
   /// Emit aliases for internal-linkage declarations inside "C" language
   /// linkage specifications, giving them the "expected" name where possible.
   void EmitStaticExternCAliases();
 
   void EmitDeclMetadata();
 
   /// Emit the Clang version as llvm.ident metadata.
   void EmitVersionIdentMetadata();
 
   /// Emit the Clang commandline as llvm.commandline metadata.
   void EmitCommandLineMetadata();
 
   /// Emit the module flag metadata used to pass options controlling the
   /// the backend to LLVM.
   void EmitBackendOptionsMetadata(const CodeGenOptions CodeGenOpts);
 
   /// Emits OpenCL specific Metadata e.g. OpenCL version.
   void EmitOpenCLMetadata();
 
   /// Emit the llvm.gcov metadata used to tell LLVM where to emit the .gcno and
   /// .gcda files in a way that persists in .bc files.
   void EmitCoverageFile();
 
   /// Determine whether the definition must be emitted; if this returns \c
   /// false, the definition can be emitted lazily if it's used.
   bool MustBeEmitted(const ValueDecl *D);
 
   /// Determine whether the definition can be emitted eagerly, or should be
   /// delayed until the end of the translation unit. This is relevant for
   /// definitions whose linkage can change, e.g. implicit function instantions
   /// which may later be explicitly instantiated.
   bool MayBeEmittedEagerly(const ValueDecl *D);
 
   /// Check whether we can use a "simpler", more core exceptions personality
   /// function.
   void SimplifyPersonality();
 
   /// Helper function for ConstructAttributeList and
   /// addDefaultFunctionDefinitionAttributes.  Builds a set of function
   /// attributes to add to a function with the given properties.
   void getDefaultFunctionAttributes(StringRef Name, bool HasOptnone,
                                     bool AttrOnCallSite,
                                     llvm::AttrBuilder &FuncAttrs);
 
   llvm::Metadata *CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map,
                                                StringRef Suffix);
 };
 
 }  // end namespace CodeGen
 }  // end namespace clang
 
 #endif // LLVM_CLANG_LIB_CODEGEN_CODEGENMODULE_H
diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/AMDGPU.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/AMDGPU.cpp
index 43ce33750eba..9638fa2ecfca 100644
--- a/contrib/llvm-project/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -1,931 +1,931 @@
 //===--- AMDGPU.cpp - AMDGPU ToolChain Implementations ----------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPU.h"
 #include "CommonArgs.h"
 #include "clang/Basic/TargetID.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/DriverDiagnostic.h"
 #include "clang/Driver/InputInfo.h"
 #include "clang/Driver/Options.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/LineIterator.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include <system_error>
 
 #define AMDGPU_ARCH_PROGRAM_NAME "amdgpu-arch"
 
 using namespace clang::driver;
 using namespace clang::driver::tools;
 using namespace clang::driver::toolchains;
 using namespace clang;
 using namespace llvm::opt;
 
 // Look for sub-directory starts with PackageName under ROCm candidate path.
 // If there is one and only one matching sub-directory found, append the
 // sub-directory to Path. If there is no matching sub-directory or there are
 // more than one matching sub-directories, diagnose them. Returns the full
 // path of the package if there is only one matching sub-directory, otherwise
 // returns an empty string.
 llvm::SmallString<0>
 RocmInstallationDetector::findSPACKPackage(const Candidate &Cand,
                                            StringRef PackageName) {
   if (!Cand.isSPACK())
     return {};
   std::error_code EC;
   std::string Prefix = Twine(PackageName + "-" + Cand.SPACKReleaseStr).str();
   llvm::SmallVector<llvm::SmallString<0>> SubDirs;
   for (llvm::vfs::directory_iterator File = D.getVFS().dir_begin(Cand.Path, EC),
                                      FileEnd;
        File != FileEnd && !EC; File.increment(EC)) {
     llvm::StringRef FileName = llvm::sys::path::filename(File->path());
     if (FileName.startswith(Prefix)) {
       SubDirs.push_back(FileName);
       if (SubDirs.size() > 1)
         break;
     }
   }
   if (SubDirs.size() == 1) {
     auto PackagePath = Cand.Path;
     llvm::sys::path::append(PackagePath, SubDirs[0]);
     return PackagePath;
   }
   if (SubDirs.size() == 0 && Verbose) {
     llvm::errs() << "SPACK package " << Prefix << " not found at " << Cand.Path
                  << '\n';
     return {};
   }
 
   if (SubDirs.size() > 1 && Verbose) {
     llvm::errs() << "Cannot use SPACK package " << Prefix << " at " << Cand.Path
                  << " due to multiple installations for the same version\n";
   }
   return {};
 }
 
 void RocmInstallationDetector::scanLibDevicePath(llvm::StringRef Path) {
   assert(!Path.empty());
 
   const StringRef Suffix(".bc");
   const StringRef Suffix2(".amdgcn.bc");
 
   std::error_code EC;
   for (llvm::vfs::directory_iterator LI = D.getVFS().dir_begin(Path, EC), LE;
        !EC && LI != LE; LI = LI.increment(EC)) {
     StringRef FilePath = LI->path();
     StringRef FileName = llvm::sys::path::filename(FilePath);
     if (!FileName.endswith(Suffix))
       continue;
 
     StringRef BaseName;
     if (FileName.endswith(Suffix2))
       BaseName = FileName.drop_back(Suffix2.size());
     else if (FileName.endswith(Suffix))
       BaseName = FileName.drop_back(Suffix.size());
 
     if (BaseName == "ocml") {
       OCML = FilePath;
     } else if (BaseName == "ockl") {
       OCKL = FilePath;
     } else if (BaseName == "opencl") {
       OpenCL = FilePath;
     } else if (BaseName == "hip") {
       HIP = FilePath;
     } else if (BaseName == "asanrtl") {
       AsanRTL = FilePath;
     } else if (BaseName == "oclc_finite_only_off") {
       FiniteOnly.Off = FilePath;
     } else if (BaseName == "oclc_finite_only_on") {
       FiniteOnly.On = FilePath;
     } else if (BaseName == "oclc_daz_opt_on") {
       DenormalsAreZero.On = FilePath;
     } else if (BaseName == "oclc_daz_opt_off") {
       DenormalsAreZero.Off = FilePath;
     } else if (BaseName == "oclc_correctly_rounded_sqrt_on") {
       CorrectlyRoundedSqrt.On = FilePath;
     } else if (BaseName == "oclc_correctly_rounded_sqrt_off") {
       CorrectlyRoundedSqrt.Off = FilePath;
     } else if (BaseName == "oclc_unsafe_math_on") {
       UnsafeMath.On = FilePath;
     } else if (BaseName == "oclc_unsafe_math_off") {
       UnsafeMath.Off = FilePath;
     } else if (BaseName == "oclc_wavefrontsize64_on") {
       WavefrontSize64.On = FilePath;
     } else if (BaseName == "oclc_wavefrontsize64_off") {
       WavefrontSize64.Off = FilePath;
     } else {
       // Process all bitcode filenames that look like
       // ocl_isa_version_XXX.amdgcn.bc
       const StringRef DeviceLibPrefix = "oclc_isa_version_";
       if (!BaseName.startswith(DeviceLibPrefix))
         continue;
 
       StringRef IsaVersionNumber =
         BaseName.drop_front(DeviceLibPrefix.size());
 
       llvm::Twine GfxName = Twine("gfx") + IsaVersionNumber;
       SmallString<8> Tmp;
       LibDeviceMap.insert(
         std::make_pair(GfxName.toStringRef(Tmp), FilePath.str()));
     }
   }
 }
 
 // Parse and extract version numbers from `.hipVersion`. Return `true` if
 // the parsing fails.
 bool RocmInstallationDetector::parseHIPVersionFile(llvm::StringRef V) {
   SmallVector<StringRef, 4> VersionParts;
   V.split(VersionParts, '\n');
   unsigned Major = ~0U;
   unsigned Minor = ~0U;
   for (auto Part : VersionParts) {
     auto Splits = Part.rtrim().split('=');
     if (Splits.first == "HIP_VERSION_MAJOR") {
       if (Splits.second.getAsInteger(0, Major))
         return true;
     } else if (Splits.first == "HIP_VERSION_MINOR") {
       if (Splits.second.getAsInteger(0, Minor))
         return true;
     } else if (Splits.first == "HIP_VERSION_PATCH")
       VersionPatch = Splits.second.str();
   }
   if (Major == ~0U || Minor == ~0U)
     return true;
   VersionMajorMinor = llvm::VersionTuple(Major, Minor);
   DetectedVersion =
       (Twine(Major) + "." + Twine(Minor) + "." + VersionPatch).str();
   return false;
 }
 
 /// \returns a list of candidate directories for ROCm installation, which is
 /// cached and populated only once.
 const SmallVectorImpl<RocmInstallationDetector::Candidate> &
 RocmInstallationDetector::getInstallationPathCandidates() {
 
   // Return the cached candidate list if it has already been populated.
   if (!ROCmSearchDirs.empty())
     return ROCmSearchDirs;
 
   auto DoPrintROCmSearchDirs = [&]() {
     if (PrintROCmSearchDirs)
       for (auto Cand : ROCmSearchDirs) {
         llvm::errs() << "ROCm installation search path";
         if (Cand.isSPACK())
           llvm::errs() << " (Spack " << Cand.SPACKReleaseStr << ")";
         llvm::errs() << ": " << Cand.Path << '\n';
       }
   };
 
   // For candidate specified by --rocm-path we do not do strict check, i.e.,
   // checking existence of HIP version file and device library files.
   if (!RocmPathArg.empty()) {
     ROCmSearchDirs.emplace_back(RocmPathArg.str());
     DoPrintROCmSearchDirs();
     return ROCmSearchDirs;
   } else if (const char *RocmPathEnv = ::getenv("ROCM_PATH")) {
     if (!StringRef(RocmPathEnv).empty()) {
       ROCmSearchDirs.emplace_back(RocmPathEnv);
       DoPrintROCmSearchDirs();
       return ROCmSearchDirs;
     }
   }
 
   // Try to find relative to the compiler binary.
   const char *InstallDir = D.getInstalledDir();
 
   // Check both a normal Unix prefix position of the clang binary, as well as
   // the Windows-esque layout the ROCm packages use with the host architecture
   // subdirectory of bin.
   auto DeduceROCmPath = [](StringRef ClangPath) {
     // Strip off directory (usually bin)
     StringRef ParentDir = llvm::sys::path::parent_path(ClangPath);
     StringRef ParentName = llvm::sys::path::filename(ParentDir);
 
     // Some builds use bin/{host arch}, so go up again.
     if (ParentName == "bin") {
       ParentDir = llvm::sys::path::parent_path(ParentDir);
       ParentName = llvm::sys::path::filename(ParentDir);
     }
 
     // Detect ROCm packages built with SPACK.
     // clang is installed at
     // <rocm_root>/llvm-amdgpu-<rocm_release_string>-<hash>/bin directory.
     // We only consider the parent directory of llvm-amdgpu package as ROCm
     // installation candidate for SPACK.
     if (ParentName.startswith("llvm-amdgpu-")) {
       auto SPACKPostfix =
           ParentName.drop_front(strlen("llvm-amdgpu-")).split('-');
       auto SPACKReleaseStr = SPACKPostfix.first;
       if (!SPACKReleaseStr.empty()) {
         ParentDir = llvm::sys::path::parent_path(ParentDir);
         return Candidate(ParentDir.str(), /*StrictChecking=*/true,
                          SPACKReleaseStr);
       }
     }
 
     // Some versions of the rocm llvm package install to /opt/rocm/llvm/bin
     // Some versions of the aomp package install to /opt/rocm/aomp/bin
     if (ParentName == "llvm" || ParentName.startswith("aomp"))
       ParentDir = llvm::sys::path::parent_path(ParentDir);
 
     return Candidate(ParentDir.str(), /*StrictChecking=*/true);
   };
 
   // Deduce ROCm path by the path used to invoke clang. Do not resolve symbolic
   // link of clang itself.
   ROCmSearchDirs.emplace_back(DeduceROCmPath(InstallDir));
 
   // Deduce ROCm path by the real path of the invoked clang, resolving symbolic
   // link of clang itself.
   llvm::SmallString<256> RealClangPath;
   llvm::sys::fs::real_path(D.getClangProgramPath(), RealClangPath);
   auto ParentPath = llvm::sys::path::parent_path(RealClangPath);
   if (ParentPath != InstallDir)
     ROCmSearchDirs.emplace_back(DeduceROCmPath(ParentPath));
 
   // Device library may be installed in clang or resource directory.
   auto ClangRoot = llvm::sys::path::parent_path(InstallDir);
   auto RealClangRoot = llvm::sys::path::parent_path(ParentPath);
   ROCmSearchDirs.emplace_back(ClangRoot.str(), /*StrictChecking=*/true);
   if (RealClangRoot != ClangRoot)
     ROCmSearchDirs.emplace_back(RealClangRoot.str(), /*StrictChecking=*/true);
   ROCmSearchDirs.emplace_back(D.ResourceDir,
                               /*StrictChecking=*/true);
 
   ROCmSearchDirs.emplace_back(D.SysRoot + "/opt/rocm",
                               /*StrictChecking=*/true);
 
   // Find the latest /opt/rocm-{release} directory.
   std::error_code EC;
   std::string LatestROCm;
   llvm::VersionTuple LatestVer;
   // Get ROCm version from ROCm directory name.
   auto GetROCmVersion = [](StringRef DirName) {
     llvm::VersionTuple V;
     std::string VerStr = DirName.drop_front(strlen("rocm-")).str();
     // The ROCm directory name follows the format of
     // rocm-{major}.{minor}.{subMinor}[-{build}]
     std::replace(VerStr.begin(), VerStr.end(), '-', '.');
     V.tryParse(VerStr);
     return V;
   };
   for (llvm::vfs::directory_iterator
            File = D.getVFS().dir_begin(D.SysRoot + "/opt", EC),
            FileEnd;
        File != FileEnd && !EC; File.increment(EC)) {
     llvm::StringRef FileName = llvm::sys::path::filename(File->path());
     if (!FileName.startswith("rocm-"))
       continue;
     if (LatestROCm.empty()) {
       LatestROCm = FileName.str();
       LatestVer = GetROCmVersion(LatestROCm);
       continue;
     }
     auto Ver = GetROCmVersion(FileName);
     if (LatestVer < Ver) {
       LatestROCm = FileName.str();
       LatestVer = Ver;
     }
   }
   if (!LatestROCm.empty())
     ROCmSearchDirs.emplace_back(D.SysRoot + "/opt/" + LatestROCm,
                                 /*StrictChecking=*/true);
 
   DoPrintROCmSearchDirs();
   return ROCmSearchDirs;
 }
 
 RocmInstallationDetector::RocmInstallationDetector(
     const Driver &D, const llvm::Triple &HostTriple,
     const llvm::opt::ArgList &Args, bool DetectHIPRuntime, bool DetectDeviceLib)
     : D(D) {
   Verbose = Args.hasArg(options::OPT_v);
   RocmPathArg = Args.getLastArgValue(clang::driver::options::OPT_rocm_path_EQ);
   PrintROCmSearchDirs =
       Args.hasArg(clang::driver::options::OPT_print_rocm_search_dirs);
   RocmDeviceLibPathArg =
       Args.getAllArgValues(clang::driver::options::OPT_rocm_device_lib_path_EQ);
   HIPPathArg = Args.getLastArgValue(clang::driver::options::OPT_hip_path_EQ);
   if (auto *A = Args.getLastArg(clang::driver::options::OPT_hip_version_EQ)) {
     HIPVersionArg = A->getValue();
     unsigned Major = ~0U;
     unsigned Minor = ~0U;
     SmallVector<StringRef, 3> Parts;
     HIPVersionArg.split(Parts, '.');
     if (Parts.size())
       Parts[0].getAsInteger(0, Major);
     if (Parts.size() > 1)
       Parts[1].getAsInteger(0, Minor);
     if (Parts.size() > 2)
       VersionPatch = Parts[2].str();
     if (VersionPatch.empty())
       VersionPatch = "0";
     if (Major != ~0U && Minor == ~0U)
       Minor = 0;
     if (Major == ~0U || Minor == ~0U)
       D.Diag(diag::err_drv_invalid_value)
           << A->getAsString(Args) << HIPVersionArg;
 
     VersionMajorMinor = llvm::VersionTuple(Major, Minor);
     DetectedVersion =
         (Twine(Major) + "." + Twine(Minor) + "." + VersionPatch).str();
   } else {
     VersionPatch = DefaultVersionPatch;
     VersionMajorMinor =
         llvm::VersionTuple(DefaultVersionMajor, DefaultVersionMinor);
     DetectedVersion = (Twine(DefaultVersionMajor) + "." +
                        Twine(DefaultVersionMinor) + "." + VersionPatch)
                           .str();
   }
 
   if (DetectHIPRuntime)
     detectHIPRuntime();
   if (DetectDeviceLib)
     detectDeviceLibrary();
 }
 
 void RocmInstallationDetector::detectDeviceLibrary() {
   assert(LibDevicePath.empty());
 
   if (!RocmDeviceLibPathArg.empty())
     LibDevicePath = RocmDeviceLibPathArg[RocmDeviceLibPathArg.size() - 1];
   else if (const char *LibPathEnv = ::getenv("HIP_DEVICE_LIB_PATH"))
     LibDevicePath = LibPathEnv;
 
   auto &FS = D.getVFS();
   if (!LibDevicePath.empty()) {
     // Maintain compatability with HIP flag/envvar pointing directly at the
     // bitcode library directory. This points directly at the library path instead
     // of the rocm root installation.
     if (!FS.exists(LibDevicePath))
       return;
 
     scanLibDevicePath(LibDevicePath);
     HasDeviceLibrary = allGenericLibsValid() && !LibDeviceMap.empty();
     return;
   }
 
   // The install path situation in old versions of ROCm is a real mess, and
   // use a different install layout. Multiple copies of the device libraries
   // exist for each frontend project, and differ depending on which build
   // system produced the packages. Standalone OpenCL builds also have a
   // different directory structure from the ROCm OpenCL package.
   auto &ROCmDirs = getInstallationPathCandidates();
   for (const auto &Candidate : ROCmDirs) {
     auto CandidatePath = Candidate.Path;
 
     // Check device library exists at the given path.
     auto CheckDeviceLib = [&](StringRef Path) {
       bool CheckLibDevice = (!NoBuiltinLibs || Candidate.StrictChecking);
       if (CheckLibDevice && !FS.exists(Path))
         return false;
 
       scanLibDevicePath(Path);
 
       if (!NoBuiltinLibs) {
         // Check that the required non-target libraries are all available.
         if (!allGenericLibsValid())
           return false;
 
         // Check that we have found at least one libdevice that we can link in
         // if -nobuiltinlib hasn't been specified.
         if (LibDeviceMap.empty())
           return false;
       }
       return true;
     };
 
     // The possible structures are:
     // - ${ROCM_ROOT}/amdgcn/bitcode/*
     // - ${ROCM_ROOT}/lib/*
     // - ${ROCM_ROOT}/lib/bitcode/*
     // so try to detect these layouts.
     static constexpr std::array<const char *, 2> SubDirsList[] = {
         {"amdgcn", "bitcode"},
         {"lib", ""},
         {"lib", "bitcode"},
     };
 
     // Make a path by appending sub-directories to InstallPath.
     auto MakePath = [&](const llvm::ArrayRef<const char *> &SubDirs) {
       auto Path = CandidatePath;
       for (auto SubDir : SubDirs)
         llvm::sys::path::append(Path, SubDir);
       return Path;
     };
 
     for (auto SubDirs : SubDirsList) {
       LibDevicePath = MakePath(SubDirs);
       HasDeviceLibrary = CheckDeviceLib(LibDevicePath);
       if (HasDeviceLibrary)
         return;
     }
   }
 }
 
 void RocmInstallationDetector::detectHIPRuntime() {
   SmallVector<Candidate, 4> HIPSearchDirs;
   if (!HIPPathArg.empty())
     HIPSearchDirs.emplace_back(HIPPathArg.str(), /*StrictChecking=*/true);
   else
     HIPSearchDirs.append(getInstallationPathCandidates());
   auto &FS = D.getVFS();
 
   for (const auto &Candidate : HIPSearchDirs) {
     InstallPath = Candidate.Path;
     if (InstallPath.empty() || !FS.exists(InstallPath))
       continue;
     // HIP runtime built by SPACK is installed to
     // <rocm_root>/hip-<rocm_release_string>-<hash> directory.
     auto SPACKPath = findSPACKPackage(Candidate, "hip");
     InstallPath = SPACKPath.empty() ? InstallPath : SPACKPath;
 
     BinPath = InstallPath;
     llvm::sys::path::append(BinPath, "bin");
     IncludePath = InstallPath;
     llvm::sys::path::append(IncludePath, "include");
     LibPath = InstallPath;
     llvm::sys::path::append(LibPath, "lib");
 
     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> VersionFile =
         FS.getBufferForFile(BinPath + "/.hipVersion");
     if (!VersionFile && Candidate.StrictChecking)
       continue;
 
     if (HIPVersionArg.empty() && VersionFile)
       if (parseHIPVersionFile((*VersionFile)->getBuffer()))
         continue;
 
     HasHIPRuntime = true;
     return;
   }
   HasHIPRuntime = false;
 }
 
 void RocmInstallationDetector::print(raw_ostream &OS) const {
   if (hasHIPRuntime())
     OS << "Found HIP installation: " << InstallPath << ", version "
        << DetectedVersion << '\n';
 }
 
 void RocmInstallationDetector::AddHIPIncludeArgs(const ArgList &DriverArgs,
                                                  ArgStringList &CC1Args) const {
   bool UsesRuntimeWrapper = VersionMajorMinor > llvm::VersionTuple(3, 5) &&
                             !DriverArgs.hasArg(options::OPT_nohipwrapperinc);
 
   if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {
     // HIP header includes standard library wrapper headers under clang
     // cuda_wrappers directory. Since these wrapper headers include_next
     // standard C++ headers, whereas libc++ headers include_next other clang
     // headers. The include paths have to follow this order:
     // - wrapper include path
     // - standard C++ include path
     // - other clang include path
     // Since standard C++ and other clang include paths are added in other
     // places after this function, here we only need to make sure wrapper
     // include path is added.
     //
     // ROCm 3.5 does not fully support the wrapper headers. Therefore it needs
     // a workaround.
     SmallString<128> P(D.ResourceDir);
     if (UsesRuntimeWrapper)
       llvm::sys::path::append(P, "include", "cuda_wrappers");
     CC1Args.push_back("-internal-isystem");
     CC1Args.push_back(DriverArgs.MakeArgString(P));
   }
 
   if (DriverArgs.hasArg(options::OPT_nogpuinc))
     return;
 
   if (!hasHIPRuntime()) {
     D.Diag(diag::err_drv_no_hip_runtime);
     return;
   }
 
-  CC1Args.push_back("-internal-isystem");
+  CC1Args.push_back("-idirafter");
   CC1Args.push_back(DriverArgs.MakeArgString(getIncludePath()));
   if (UsesRuntimeWrapper)
     CC1Args.append({"-include", "__clang_hip_runtime_wrapper.h"});
 }
 
 void amdgpu::Linker::ConstructJob(Compilation &C, const JobAction &JA,
                                   const InputInfo &Output,
                                   const InputInfoList &Inputs,
                                   const ArgList &Args,
                                   const char *LinkingOutput) const {
 
   std::string Linker = getToolChain().GetProgramPath(getShortName());
   ArgStringList CmdArgs;
   addLinkerCompressDebugSectionsOption(getToolChain(), Args, CmdArgs);
   AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs, JA);
   CmdArgs.push_back("-shared");
   CmdArgs.push_back("-o");
   CmdArgs.push_back(Output.getFilename());
   C.addCommand(std::make_unique<Command>(
       JA, *this, ResponseFileSupport::AtFileCurCP(), Args.MakeArgString(Linker),
       CmdArgs, Inputs, Output));
 }
 
 void amdgpu::getAMDGPUTargetFeatures(const Driver &D,
                                      const llvm::Triple &Triple,
                                      const llvm::opt::ArgList &Args,
                                      std::vector<StringRef> &Features) {
   // Add target ID features to -target-feature options. No diagnostics should
   // be emitted here since invalid target ID is diagnosed at other places.
   StringRef TargetID = Args.getLastArgValue(options::OPT_mcpu_EQ);
   if (!TargetID.empty()) {
     llvm::StringMap<bool> FeatureMap;
     auto OptionalGpuArch = parseTargetID(Triple, TargetID, &FeatureMap);
     if (OptionalGpuArch) {
       StringRef GpuArch = OptionalGpuArch.getValue();
       // Iterate through all possible target ID features for the given GPU.
       // If it is mapped to true, add +feature.
       // If it is mapped to false, add -feature.
       // If it is not in the map (default), do not add it
       for (auto &&Feature : getAllPossibleTargetIDFeatures(Triple, GpuArch)) {
         auto Pos = FeatureMap.find(Feature);
         if (Pos == FeatureMap.end())
           continue;
         Features.push_back(Args.MakeArgStringRef(
             (Twine(Pos->second ? "+" : "-") + Feature).str()));
       }
     }
   }
 
   if (Args.hasFlag(options::OPT_mwavefrontsize64,
                    options::OPT_mno_wavefrontsize64, false))
     Features.push_back("+wavefrontsize64");
 
   handleTargetFeaturesGroup(
     Args, Features, options::OPT_m_amdgpu_Features_Group);
 }
 
 /// AMDGPU Toolchain
 AMDGPUToolChain::AMDGPUToolChain(const Driver &D, const llvm::Triple &Triple,
                                  const ArgList &Args)
     : Generic_ELF(D, Triple, Args),
       OptionsDefault(
           {{options::OPT_O, "3"}, {options::OPT_cl_std_EQ, "CL1.2"}}) {
   // Check code object version options. Emit warnings for legacy options
   // and errors for the last invalid code object version options.
   // It is done here to avoid repeated warning or error messages for
   // each tool invocation.
   checkAMDGPUCodeObjectVersion(D, Args);
 }
 
 Tool *AMDGPUToolChain::buildLinker() const {
   return new tools::amdgpu::Linker(*this);
 }
 
 DerivedArgList *
 AMDGPUToolChain::TranslateArgs(const DerivedArgList &Args, StringRef BoundArch,
                                Action::OffloadKind DeviceOffloadKind) const {
 
   DerivedArgList *DAL =
       Generic_ELF::TranslateArgs(Args, BoundArch, DeviceOffloadKind);
 
   const OptTable &Opts = getDriver().getOpts();
 
   if (!DAL)
     DAL = new DerivedArgList(Args.getBaseArgs());
 
   for (Arg *A : Args) {
     if (!shouldSkipArgument(A))
       DAL->append(A);
   }
 
   checkTargetID(*DAL);
 
   if (!Args.getLastArgValue(options::OPT_x).equals("cl"))
     return DAL;
 
   // Phase 1 (.cl -> .bc)
   if (Args.hasArg(options::OPT_c) && Args.hasArg(options::OPT_emit_llvm)) {
     DAL->AddFlagArg(nullptr, Opts.getOption(getTriple().isArch64Bit()
                                                 ? options::OPT_m64
                                                 : options::OPT_m32));
 
     // Have to check OPT_O4, OPT_O0 & OPT_Ofast separately
     // as they defined that way in Options.td
     if (!Args.hasArg(options::OPT_O, options::OPT_O0, options::OPT_O4,
                      options::OPT_Ofast))
       DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_O),
                         getOptionDefault(options::OPT_O));
   }
 
   return DAL;
 }
 
 bool AMDGPUToolChain::getDefaultDenormsAreZeroForTarget(
     llvm::AMDGPU::GPUKind Kind) {
 
   // Assume nothing without a specific target.
   if (Kind == llvm::AMDGPU::GK_NONE)
     return false;
 
   const unsigned ArchAttr = llvm::AMDGPU::getArchAttrAMDGCN(Kind);
 
   // Default to enabling f32 denormals by default on subtargets where fma is
   // fast with denormals
   const bool BothDenormAndFMAFast =
       (ArchAttr & llvm::AMDGPU::FEATURE_FAST_FMA_F32) &&
       (ArchAttr & llvm::AMDGPU::FEATURE_FAST_DENORMAL_F32);
   return !BothDenormAndFMAFast;
 }
 
 llvm::DenormalMode AMDGPUToolChain::getDefaultDenormalModeForType(
     const llvm::opt::ArgList &DriverArgs, const JobAction &JA,
     const llvm::fltSemantics *FPType) const {
   // Denormals should always be enabled for f16 and f64.
   if (!FPType || FPType != &llvm::APFloat::IEEEsingle())
     return llvm::DenormalMode::getIEEE();
 
   if (JA.getOffloadingDeviceKind() == Action::OFK_HIP ||
       JA.getOffloadingDeviceKind() == Action::OFK_Cuda) {
     auto Arch = getProcessorFromTargetID(getTriple(), JA.getOffloadingArch());
     auto Kind = llvm::AMDGPU::parseArchAMDGCN(Arch);
     if (FPType && FPType == &llvm::APFloat::IEEEsingle() &&
         DriverArgs.hasFlag(options::OPT_fgpu_flush_denormals_to_zero,
                            options::OPT_fno_gpu_flush_denormals_to_zero,
                            getDefaultDenormsAreZeroForTarget(Kind)))
       return llvm::DenormalMode::getPreserveSign();
 
     return llvm::DenormalMode::getIEEE();
   }
 
   const StringRef GpuArch = getGPUArch(DriverArgs);
   auto Kind = llvm::AMDGPU::parseArchAMDGCN(GpuArch);
 
   // TODO: There are way too many flags that change this. Do we need to check
   // them all?
   bool DAZ = DriverArgs.hasArg(options::OPT_cl_denorms_are_zero) ||
              getDefaultDenormsAreZeroForTarget(Kind);
 
   // Outputs are flushed to zero (FTZ), preserving sign. Denormal inputs are
   // also implicit treated as zero (DAZ).
   return DAZ ? llvm::DenormalMode::getPreserveSign() :
                llvm::DenormalMode::getIEEE();
 }
 
 bool AMDGPUToolChain::isWave64(const llvm::opt::ArgList &DriverArgs,
                                llvm::AMDGPU::GPUKind Kind) {
   const unsigned ArchAttr = llvm::AMDGPU::getArchAttrAMDGCN(Kind);
   bool HasWave32 = (ArchAttr & llvm::AMDGPU::FEATURE_WAVE32);
 
   return !HasWave32 || DriverArgs.hasFlag(
     options::OPT_mwavefrontsize64, options::OPT_mno_wavefrontsize64, false);
 }
 
 
 /// ROCM Toolchain
 ROCMToolChain::ROCMToolChain(const Driver &D, const llvm::Triple &Triple,
                              const ArgList &Args)
     : AMDGPUToolChain(D, Triple, Args) {
   RocmInstallation.detectDeviceLibrary();
 }
 
 void AMDGPUToolChain::addClangTargetOptions(
     const llvm::opt::ArgList &DriverArgs,
     llvm::opt::ArgStringList &CC1Args,
     Action::OffloadKind DeviceOffloadingKind) const {
   // Default to "hidden" visibility, as object level linking will not be
   // supported for the foreseeable future.
   if (!DriverArgs.hasArg(options::OPT_fvisibility_EQ,
                          options::OPT_fvisibility_ms_compat)) {
     CC1Args.push_back("-fvisibility");
     CC1Args.push_back("hidden");
     CC1Args.push_back("-fapply-global-visibility-to-externs");
   }
 }
 
 StringRef
 AMDGPUToolChain::getGPUArch(const llvm::opt::ArgList &DriverArgs) const {
   return getProcessorFromTargetID(
       getTriple(), DriverArgs.getLastArgValue(options::OPT_mcpu_EQ));
 }
 
 AMDGPUToolChain::ParsedTargetIDType
 AMDGPUToolChain::getParsedTargetID(const llvm::opt::ArgList &DriverArgs) const {
   StringRef TargetID = DriverArgs.getLastArgValue(options::OPT_mcpu_EQ);
   if (TargetID.empty())
     return {None, None, None};
 
   llvm::StringMap<bool> FeatureMap;
   auto OptionalGpuArch = parseTargetID(getTriple(), TargetID, &FeatureMap);
   if (!OptionalGpuArch)
     return {TargetID.str(), None, None};
 
   return {TargetID.str(), OptionalGpuArch.getValue().str(), FeatureMap};
 }
 
 void AMDGPUToolChain::checkTargetID(
     const llvm::opt::ArgList &DriverArgs) const {
   auto PTID = getParsedTargetID(DriverArgs);
   if (PTID.OptionalTargetID && !PTID.OptionalGPUArch) {
     getDriver().Diag(clang::diag::err_drv_bad_target_id)
         << PTID.OptionalTargetID.getValue();
   }
 }
 
 llvm::Error
 AMDGPUToolChain::detectSystemGPUs(const ArgList &Args,
                                   SmallVector<std::string, 1> &GPUArchs) const {
   std::string Program;
   if (Arg *A = Args.getLastArg(options::OPT_amdgpu_arch_tool_EQ))
     Program = A->getValue();
   else
     Program = GetProgramPath(AMDGPU_ARCH_PROGRAM_NAME);
   llvm::SmallString<64> OutputFile;
   llvm::sys::fs::createTemporaryFile("print-system-gpus", "" /* No Suffix */,
                                      OutputFile);
   llvm::FileRemover OutputRemover(OutputFile.c_str());
   llvm::Optional<llvm::StringRef> Redirects[] = {
       {""},
       OutputFile.str(),
       {""},
   };
 
   std::string ErrorMessage;
   if (int Result = llvm::sys::ExecuteAndWait(
           Program, {}, {}, Redirects, /* SecondsToWait */ 0,
           /*MemoryLimit*/ 0, &ErrorMessage)) {
     if (Result > 0) {
       ErrorMessage = "Exited with error code " + std::to_string(Result);
     } else if (Result == -1) {
       ErrorMessage = "Execute failed: " + ErrorMessage;
     } else {
       ErrorMessage = "Crashed: " + ErrorMessage;
     }
 
     return llvm::createStringError(std::error_code(),
                                    Program + ": " + ErrorMessage);
   }
 
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> OutputBuf =
       llvm::MemoryBuffer::getFile(OutputFile.c_str());
   if (!OutputBuf) {
     return llvm::createStringError(OutputBuf.getError(),
                                    "Failed to read stdout of " + Program +
                                        ": " + OutputBuf.getError().message());
   }
 
   for (llvm::line_iterator LineIt(**OutputBuf); !LineIt.is_at_end(); ++LineIt) {
     GPUArchs.push_back(LineIt->str());
   }
   return llvm::Error::success();
 }
 
 llvm::Error AMDGPUToolChain::getSystemGPUArch(const ArgList &Args,
                                               std::string &GPUArch) const {
   // detect the AMDGPU installed in system
   SmallVector<std::string, 1> GPUArchs;
   auto Err = detectSystemGPUs(Args, GPUArchs);
   if (Err) {
     return Err;
   }
   if (GPUArchs.empty()) {
     return llvm::createStringError(std::error_code(),
                                    "No AMD GPU detected in the system");
   }
   GPUArch = GPUArchs[0];
   if (GPUArchs.size() > 1) {
     bool AllSame = llvm::all_of(GPUArchs, [&](const StringRef &GPUArch) {
       return GPUArch == GPUArchs.front();
     });
     if (!AllSame)
       return llvm::createStringError(
           std::error_code(), "Multiple AMD GPUs found with different archs");
   }
   return llvm::Error::success();
 }
 
 void ROCMToolChain::addClangTargetOptions(
     const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
     Action::OffloadKind DeviceOffloadingKind) const {
   AMDGPUToolChain::addClangTargetOptions(DriverArgs, CC1Args,
                                          DeviceOffloadingKind);
 
   // For the OpenCL case where there is no offload target, accept -nostdlib to
   // disable bitcode linking.
   if (DeviceOffloadingKind == Action::OFK_None &&
       DriverArgs.hasArg(options::OPT_nostdlib))
     return;
 
   if (DriverArgs.hasArg(options::OPT_nogpulib))
     return;
 
   if (!RocmInstallation.hasDeviceLibrary()) {
     getDriver().Diag(diag::err_drv_no_rocm_device_lib) << 0;
     return;
   }
 
   // Get the device name and canonicalize it
   const StringRef GpuArch = getGPUArch(DriverArgs);
   auto Kind = llvm::AMDGPU::parseArchAMDGCN(GpuArch);
   const StringRef CanonArch = llvm::AMDGPU::getArchNameAMDGCN(Kind);
   std::string LibDeviceFile = RocmInstallation.getLibDeviceFile(CanonArch);
   if (LibDeviceFile.empty()) {
     getDriver().Diag(diag::err_drv_no_rocm_device_lib) << 1 << GpuArch;
     return;
   }
 
   bool Wave64 = isWave64(DriverArgs, Kind);
 
   // TODO: There are way too many flags that change this. Do we need to check
   // them all?
   bool DAZ = DriverArgs.hasArg(options::OPT_cl_denorms_are_zero) ||
              getDefaultDenormsAreZeroForTarget(Kind);
   bool FiniteOnly = DriverArgs.hasArg(options::OPT_cl_finite_math_only);
 
   bool UnsafeMathOpt =
       DriverArgs.hasArg(options::OPT_cl_unsafe_math_optimizations);
   bool FastRelaxedMath = DriverArgs.hasArg(options::OPT_cl_fast_relaxed_math);
   bool CorrectSqrt =
       DriverArgs.hasArg(options::OPT_cl_fp32_correctly_rounded_divide_sqrt);
 
   // Add the OpenCL specific bitcode library.
   llvm::SmallVector<std::string, 12> BCLibs;
   BCLibs.push_back(RocmInstallation.getOpenCLPath().str());
 
   // Add the generic set of libraries.
   BCLibs.append(RocmInstallation.getCommonBitcodeLibs(
       DriverArgs, LibDeviceFile, Wave64, DAZ, FiniteOnly, UnsafeMathOpt,
       FastRelaxedMath, CorrectSqrt));
 
   llvm::for_each(BCLibs, [&](StringRef BCFile) {
     CC1Args.push_back("-mlink-builtin-bitcode");
     CC1Args.push_back(DriverArgs.MakeArgString(BCFile));
   });
 }
 
 llvm::SmallVector<std::string, 12>
 RocmInstallationDetector::getCommonBitcodeLibs(
     const llvm::opt::ArgList &DriverArgs, StringRef LibDeviceFile, bool Wave64,
     bool DAZ, bool FiniteOnly, bool UnsafeMathOpt, bool FastRelaxedMath,
     bool CorrectSqrt) const {
 
   llvm::SmallVector<std::string, 12> BCLibs;
 
   auto AddBCLib = [&](StringRef BCFile) { BCLibs.push_back(BCFile.str()); };
 
   AddBCLib(getOCMLPath());
   AddBCLib(getOCKLPath());
   AddBCLib(getDenormalsAreZeroPath(DAZ));
   AddBCLib(getUnsafeMathPath(UnsafeMathOpt || FastRelaxedMath));
   AddBCLib(getFiniteOnlyPath(FiniteOnly || FastRelaxedMath));
   AddBCLib(getCorrectlyRoundedSqrtPath(CorrectSqrt));
   AddBCLib(getWavefrontSize64Path(Wave64));
   AddBCLib(LibDeviceFile);
 
   return BCLibs;
 }
 
 bool AMDGPUToolChain::shouldSkipArgument(const llvm::opt::Arg *A) const {
   Option O = A->getOption();
   if (O.matches(options::OPT_fPIE) || O.matches(options::OPT_fpie))
     return true;
   return false;
 }
 
 llvm::SmallVector<std::string, 12>
 ROCMToolChain::getCommonDeviceLibNames(const llvm::opt::ArgList &DriverArgs,
                                        const std::string &GPUArch) const {
   auto Kind = llvm::AMDGPU::parseArchAMDGCN(GPUArch);
   const StringRef CanonArch = llvm::AMDGPU::getArchNameAMDGCN(Kind);
 
   std::string LibDeviceFile = RocmInstallation.getLibDeviceFile(CanonArch);
   if (LibDeviceFile.empty()) {
     getDriver().Diag(diag::err_drv_no_rocm_device_lib) << 1 << GPUArch;
     return {};
   }
 
   // If --hip-device-lib is not set, add the default bitcode libraries.
   // TODO: There are way too many flags that change this. Do we need to check
   // them all?
   bool DAZ = DriverArgs.hasFlag(options::OPT_fgpu_flush_denormals_to_zero,
                                 options::OPT_fno_gpu_flush_denormals_to_zero,
                                 getDefaultDenormsAreZeroForTarget(Kind));
   bool FiniteOnly = DriverArgs.hasFlag(
       options::OPT_ffinite_math_only, options::OPT_fno_finite_math_only, false);
   bool UnsafeMathOpt =
       DriverArgs.hasFlag(options::OPT_funsafe_math_optimizations,
                          options::OPT_fno_unsafe_math_optimizations, false);
   bool FastRelaxedMath = DriverArgs.hasFlag(options::OPT_ffast_math,
                                             options::OPT_fno_fast_math, false);
   bool CorrectSqrt = DriverArgs.hasFlag(
       options::OPT_fhip_fp32_correctly_rounded_divide_sqrt,
       options::OPT_fno_hip_fp32_correctly_rounded_divide_sqrt);
   bool Wave64 = isWave64(DriverArgs, Kind);
 
   return RocmInstallation.getCommonBitcodeLibs(
       DriverArgs, LibDeviceFile, Wave64, DAZ, FiniteOnly, UnsafeMathOpt,
       FastRelaxedMath, CorrectSqrt);
 }
diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/Ananas.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/Ananas.cpp
index be1476a7636c..40f9e56b38e9 100644
--- a/contrib/llvm-project/clang/lib/Driver/ToolChains/Ananas.cpp
+++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/Ananas.cpp
@@ -1,144 +1,148 @@
 //===--- Ananas.cpp - Ananas ToolChain Implementations ------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "Ananas.h"
 #include "CommonArgs.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/InputInfo.h"
 #include "clang/Driver/Options.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/Path.h"
 
 using namespace clang::driver;
 using namespace clang::driver::tools;
 using namespace clang::driver::toolchains;
 using namespace clang;
 using namespace llvm::opt;
 
 void ananas::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
                                      const InputInfo &Output,
                                      const InputInfoList &Inputs,
                                      const ArgList &Args,
                                      const char *LinkingOutput) const {
   claimNoWarnArgs(Args);
   ArgStringList CmdArgs;
 
   Args.AddAllArgValues(CmdArgs, options::OPT_Wa_COMMA, options::OPT_Xassembler);
 
   CmdArgs.push_back("-o");
   CmdArgs.push_back(Output.getFilename());
 
   for (const auto &II : Inputs)
     CmdArgs.push_back(II.getFilename());
 
   const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("as"));
   C.addCommand(std::make_unique<Command>(JA, *this,
                                          ResponseFileSupport::AtFileCurCP(),
                                          Exec, CmdArgs, Inputs, Output));
 }
 
 void ananas::Linker::ConstructJob(Compilation &C, const JobAction &JA,
                                   const InputInfo &Output,
                                   const InputInfoList &Inputs,
                                   const ArgList &Args,
                                   const char *LinkingOutput) const {
   const ToolChain &ToolChain = getToolChain();
   const Driver &D = ToolChain.getDriver();
   ArgStringList CmdArgs;
 
   // Silence warning for "clang -g foo.o -o foo"
   Args.ClaimAllArgs(options::OPT_g_Group);
   // and "clang -emit-llvm foo.o -o foo"
   Args.ClaimAllArgs(options::OPT_emit_llvm);
   // and for "clang -w foo.o -o foo". Other warning options are already
   // handled somewhere else.
   Args.ClaimAllArgs(options::OPT_w);
 
   if (!D.SysRoot.empty())
     CmdArgs.push_back(Args.MakeArgString("--sysroot=" + D.SysRoot));
 
   if (Args.hasArg(options::OPT_static)) {
     CmdArgs.push_back("-Bstatic");
   } else {
     if (Args.hasArg(options::OPT_rdynamic))
       CmdArgs.push_back("-export-dynamic");
     if (Args.hasArg(options::OPT_shared)) {
       CmdArgs.push_back("-Bshareable");
     } else {
       Args.AddAllArgs(CmdArgs, options::OPT_pie);
       CmdArgs.push_back("-dynamic-linker");
       CmdArgs.push_back("/lib/ld-ananas.so");
     }
   }
 
   if (Output.isFilename()) {
     CmdArgs.push_back("-o");
     CmdArgs.push_back(Output.getFilename());
   } else {
     assert(Output.isNothing() && "Invalid output.");
   }
 
-  if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles)) {
+  if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles,
+                   options::OPT_r)) {
     if (!Args.hasArg(options::OPT_shared)) {
       CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crt0.o")));
     }
     CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crti.o")));
     if (Args.hasArg(options::OPT_shared) || Args.hasArg(options::OPT_pie)) {
       CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crtbeginS.o")));
     } else {
       CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crtbegin.o")));
     }
   }
 
   Args.AddAllArgs(CmdArgs, options::OPT_L);
   ToolChain.AddFilePathLibArgs(Args, CmdArgs);
   Args.AddAllArgs(CmdArgs,
                   {options::OPT_T_Group, options::OPT_e, options::OPT_s,
                    options::OPT_t, options::OPT_Z_Flag, options::OPT_r});
 
   if (D.isUsingLTO()) {
     assert(!Inputs.empty() && "Must have at least one input.");
     addLTOOptions(ToolChain, Args, CmdArgs, Output, Inputs[0],
                   D.getLTOMode() == LTOK_Thin);
   }
 
   AddLinkerInputs(ToolChain, Inputs, Args, CmdArgs, JA);
 
-  if (ToolChain.ShouldLinkCXXStdlib(Args))
-    ToolChain.AddCXXStdlibLibArgs(Args, CmdArgs);
-  if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs))
+  if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs,
+                   options::OPT_r)) {
+    if (ToolChain.ShouldLinkCXXStdlib(Args))
+      ToolChain.AddCXXStdlibLibArgs(Args, CmdArgs);
     CmdArgs.push_back("-lc");
+  }
 
-  if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles)) {
+  if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles,
+                   options::OPT_r)) {
     if (Args.hasArg(options::OPT_shared) || Args.hasArg(options::OPT_pie))
       CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crtendS.o")));
     else
       CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crtend.o")));
     CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crtn.o")));
   }
 
   const char *Exec = Args.MakeArgString(ToolChain.GetLinkerPath());
   C.addCommand(std::make_unique<Command>(JA, *this,
                                          ResponseFileSupport::AtFileCurCP(),
                                          Exec, CmdArgs, Inputs, Output));
 }
 
 // Ananas - Ananas tool chain which can call as(1) and ld(1) directly.
 
 Ananas::Ananas(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
     : Generic_ELF(D, Triple, Args) {
   getFilePaths().push_back(getDriver().SysRoot + "/usr/lib");
 }
 
 Tool *Ananas::buildAssembler() const {
   return new tools::ananas::Assembler(*this);
 }
 
 Tool *Ananas::buildLinker() const { return new tools::ananas::Linker(*this); }
diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp
index dfcef2304040..8f9244cae8db 100644
--- a/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1,2053 +1,2065 @@
 //===--- CommonArgs.cpp - Args handling for multiple toolchains -*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "CommonArgs.h"
 #include "Arch/AArch64.h"
 #include "Arch/ARM.h"
 #include "Arch/M68k.h"
 #include "Arch/Mips.h"
 #include "Arch/PPC.h"
 #include "Arch/SystemZ.h"
 #include "Arch/VE.h"
 #include "Arch/X86.h"
 #include "HIPAMD.h"
 #include "Hexagon.h"
 #include "clang/Basic/CharInfo.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/ObjCRuntime.h"
 #include "clang/Basic/Version.h"
 #include "clang/Config/config.h"
 #include "clang/Driver/Action.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/DriverDiagnostic.h"
 #include "clang/Driver/InputInfo.h"
 #include "clang/Driver/Job.h"
 #include "clang/Driver/Options.h"
 #include "clang/Driver/SanitizerArgs.h"
 #include "clang/Driver/ToolChain.h"
 #include "clang/Driver/Util.h"
 #include "clang/Driver/XRayArgs.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/TargetParser.h"
 #include "llvm/Support/Threading.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/Support/YAMLParser.h"
 
 using namespace clang::driver;
 using namespace clang::driver::tools;
 using namespace clang;
 using namespace llvm::opt;
 
 static void renderRpassOptions(const ArgList &Args, ArgStringList &CmdArgs) {
   if (const Arg *A = Args.getLastArg(options::OPT_Rpass_EQ))
     CmdArgs.push_back(Args.MakeArgString(Twine("--plugin-opt=-pass-remarks=") +
                                          A->getValue()));
 
   if (const Arg *A = Args.getLastArg(options::OPT_Rpass_missed_EQ))
     CmdArgs.push_back(Args.MakeArgString(
         Twine("--plugin-opt=-pass-remarks-missed=") + A->getValue()));
 
   if (const Arg *A = Args.getLastArg(options::OPT_Rpass_analysis_EQ))
     CmdArgs.push_back(Args.MakeArgString(
         Twine("--plugin-opt=-pass-remarks-analysis=") + A->getValue()));
 }
 
 static void renderRemarksOptions(const ArgList &Args, ArgStringList &CmdArgs,
                                  const llvm::Triple &Triple,
                                  const InputInfo &Input,
                                  const InputInfo &Output) {
   StringRef Format = "yaml";
   if (const Arg *A = Args.getLastArg(options::OPT_fsave_optimization_record_EQ))
     Format = A->getValue();
 
   SmallString<128> F;
   const Arg *A = Args.getLastArg(options::OPT_foptimization_record_file_EQ);
   if (A)
     F = A->getValue();
   else if (Output.isFilename())
     F = Output.getFilename();
 
   assert(!F.empty() && "Cannot determine remarks output name.");
   // Append "opt.ld.<format>" to the end of the file name.
   CmdArgs.push_back(
       Args.MakeArgString(Twine("--plugin-opt=opt-remarks-filename=") + F +
                          Twine(".opt.ld.") + Format));
 
   if (const Arg *A =
           Args.getLastArg(options::OPT_foptimization_record_passes_EQ))
     CmdArgs.push_back(Args.MakeArgString(
         Twine("--plugin-opt=opt-remarks-passes=") + A->getValue()));
 
   CmdArgs.push_back(Args.MakeArgString(
       Twine("--plugin-opt=opt-remarks-format=") + Format.data()));
 }
 
 static void renderRemarksHotnessOptions(const ArgList &Args,
                                         ArgStringList &CmdArgs) {
   if (Args.hasFlag(options::OPT_fdiagnostics_show_hotness,
                    options::OPT_fno_diagnostics_show_hotness, false))
     CmdArgs.push_back("--plugin-opt=opt-remarks-with-hotness");
 
   if (const Arg *A =
           Args.getLastArg(options::OPT_fdiagnostics_hotness_threshold_EQ))
     CmdArgs.push_back(Args.MakeArgString(
         Twine("--plugin-opt=opt-remarks-hotness-threshold=") + A->getValue()));
 }
 
 void tools::addPathIfExists(const Driver &D, const Twine &Path,
                             ToolChain::path_list &Paths) {
   if (D.getVFS().exists(Path))
     Paths.push_back(Path.str());
 }
 
 void tools::handleTargetFeaturesGroup(const ArgList &Args,
                                       std::vector<StringRef> &Features,
                                       OptSpecifier Group) {
   for (const Arg *A : Args.filtered(Group)) {
     StringRef Name = A->getOption().getName();
     A->claim();
 
     // Skip over "-m".
     assert(Name.startswith("m") && "Invalid feature name.");
     Name = Name.substr(1);
 
     bool IsNegative = Name.startswith("no-");
     if (IsNegative)
       Name = Name.substr(3);
     Features.push_back(Args.MakeArgString((IsNegative ? "-" : "+") + Name));
   }
 }
 
 std::vector<StringRef>
 tools::unifyTargetFeatures(const std::vector<StringRef> &Features) {
   std::vector<StringRef> UnifiedFeatures;
   // Find the last of each feature.
   llvm::StringMap<unsigned> LastOpt;
   for (unsigned I = 0, N = Features.size(); I < N; ++I) {
     StringRef Name = Features[I];
     assert(Name[0] == '-' || Name[0] == '+');
     LastOpt[Name.drop_front(1)] = I;
   }
 
   for (unsigned I = 0, N = Features.size(); I < N; ++I) {
     // If this feature was overridden, ignore it.
     StringRef Name = Features[I];
     llvm::StringMap<unsigned>::iterator LastI = LastOpt.find(Name.drop_front(1));
     assert(LastI != LastOpt.end());
     unsigned Last = LastI->second;
     if (Last != I)
       continue;
 
     UnifiedFeatures.push_back(Name);
   }
   return UnifiedFeatures;
 }
 
 void tools::addDirectoryList(const ArgList &Args, ArgStringList &CmdArgs,
                              const char *ArgName, const char *EnvVar) {
   const char *DirList = ::getenv(EnvVar);
   bool CombinedArg = false;
 
   if (!DirList)
     return; // Nothing to do.
 
   StringRef Name(ArgName);
   if (Name.equals("-I") || Name.equals("-L") || Name.empty())
     CombinedArg = true;
 
   StringRef Dirs(DirList);
   if (Dirs.empty()) // Empty string should not add '.'.
     return;
 
   StringRef::size_type Delim;
   while ((Delim = Dirs.find(llvm::sys::EnvPathSeparator)) != StringRef::npos) {
     if (Delim == 0) { // Leading colon.
       if (CombinedArg) {
         CmdArgs.push_back(Args.MakeArgString(std::string(ArgName) + "."));
       } else {
         CmdArgs.push_back(ArgName);
         CmdArgs.push_back(".");
       }
     } else {
       if (CombinedArg) {
         CmdArgs.push_back(
             Args.MakeArgString(std::string(ArgName) + Dirs.substr(0, Delim)));
       } else {
         CmdArgs.push_back(ArgName);
         CmdArgs.push_back(Args.MakeArgString(Dirs.substr(0, Delim)));
       }
     }
     Dirs = Dirs.substr(Delim + 1);
   }
 
   if (Dirs.empty()) { // Trailing colon.
     if (CombinedArg) {
       CmdArgs.push_back(Args.MakeArgString(std::string(ArgName) + "."));
     } else {
       CmdArgs.push_back(ArgName);
       CmdArgs.push_back(".");
     }
   } else { // Add the last path.
     if (CombinedArg) {
       CmdArgs.push_back(Args.MakeArgString(std::string(ArgName) + Dirs));
     } else {
       CmdArgs.push_back(ArgName);
       CmdArgs.push_back(Args.MakeArgString(Dirs));
     }
   }
 }
 
 void tools::AddLinkerInputs(const ToolChain &TC, const InputInfoList &Inputs,
                             const ArgList &Args, ArgStringList &CmdArgs,
                             const JobAction &JA) {
   const Driver &D = TC.getDriver();
 
   // Add extra linker input arguments which are not treated as inputs
   // (constructed via -Xarch_).
   Args.AddAllArgValues(CmdArgs, options::OPT_Zlinker_input);
 
   // LIBRARY_PATH are included before user inputs and only supported on native
   // toolchains.
   if (!TC.isCrossCompiling())
     addDirectoryList(Args, CmdArgs, "-L", "LIBRARY_PATH");
 
   for (const auto &II : Inputs) {
     // If the current tool chain refers to an OpenMP offloading host, we
     // should ignore inputs that refer to OpenMP offloading devices -
     // they will be embedded according to a proper linker script.
     if (auto *IA = II.getAction())
       if ((JA.isHostOffloading(Action::OFK_OpenMP) &&
            IA->isDeviceOffloading(Action::OFK_OpenMP)))
         continue;
 
     if (!TC.HasNativeLLVMSupport() && types::isLLVMIR(II.getType()))
       // Don't try to pass LLVM inputs unless we have native support.
       D.Diag(diag::err_drv_no_linker_llvm_support) << TC.getTripleString();
 
     // Add filenames immediately.
     if (II.isFilename()) {
       CmdArgs.push_back(II.getFilename());
       continue;
     }
 
     // Otherwise, this is a linker input argument.
     const Arg &A = II.getInputArg();
 
     // Handle reserved library options.
     if (A.getOption().matches(options::OPT_Z_reserved_lib_stdcxx))
       TC.AddCXXStdlibLibArgs(Args, CmdArgs);
     else if (A.getOption().matches(options::OPT_Z_reserved_lib_cckext))
       TC.AddCCKextLibArgs(Args, CmdArgs);
     else if (A.getOption().matches(options::OPT_z)) {
       // Pass -z prefix for gcc linker compatibility.
       A.claim();
       A.render(Args, CmdArgs);
     } else if (A.getOption().matches(options::OPT_b)) {
       const llvm::Triple &T = TC.getTriple();
       if (!T.isOSAIX()) {
         TC.getDriver().Diag(diag::err_drv_unsupported_opt_for_target)
             << A.getSpelling() << T.str();
       }
       // Pass -b prefix for AIX linker.
       A.claim();
       A.render(Args, CmdArgs);
     } else {
       A.renderAsInput(Args, CmdArgs);
     }
   }
 }
 
 void tools::addLinkerCompressDebugSectionsOption(
     const ToolChain &TC, const llvm::opt::ArgList &Args,
     llvm::opt::ArgStringList &CmdArgs) {
   // GNU ld supports --compress-debug-sections=none|zlib|zlib-gnu|zlib-gabi
   // whereas zlib is an alias to zlib-gabi and zlib-gnu is obsoleted. Therefore
   // -gz=none|zlib are translated to --compress-debug-sections=none|zlib. -gz
   // is not translated since ld --compress-debug-sections option requires an
   // argument.
   if (const Arg *A = Args.getLastArg(options::OPT_gz_EQ)) {
     StringRef V = A->getValue();
     if (V == "none" || V == "zlib")
       CmdArgs.push_back(Args.MakeArgString("--compress-debug-sections=" + V));
     else
       TC.getDriver().Diag(diag::err_drv_unsupported_option_argument)
           << A->getOption().getName() << V;
   }
 }
 
 void tools::AddTargetFeature(const ArgList &Args,
                              std::vector<StringRef> &Features,
                              OptSpecifier OnOpt, OptSpecifier OffOpt,
                              StringRef FeatureName) {
   if (Arg *A = Args.getLastArg(OnOpt, OffOpt)) {
     if (A->getOption().matches(OnOpt))
       Features.push_back(Args.MakeArgString("+" + FeatureName));
     else
       Features.push_back(Args.MakeArgString("-" + FeatureName));
   }
 }
 
 /// Get the (LLVM) name of the AMDGPU gpu we are targeting.
 static std::string getAMDGPUTargetGPU(const llvm::Triple &T,
                                       const ArgList &Args) {
   if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ)) {
     auto GPUName = getProcessorFromTargetID(T, A->getValue());
     return llvm::StringSwitch<std::string>(GPUName)
         .Cases("rv630", "rv635", "r600")
         .Cases("rv610", "rv620", "rs780", "rs880")
         .Case("rv740", "rv770")
         .Case("palm", "cedar")
         .Cases("sumo", "sumo2", "sumo")
         .Case("hemlock", "cypress")
         .Case("aruba", "cayman")
         .Default(GPUName.str());
   }
   return "";
 }
 
 static std::string getLanaiTargetCPU(const ArgList &Args) {
   if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ)) {
     return A->getValue();
   }
   return "";
 }
 
 /// Get the (LLVM) name of the WebAssembly cpu we are targeting.
 static StringRef getWebAssemblyTargetCPU(const ArgList &Args) {
   // If we have -mcpu=, use that.
   if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ)) {
     StringRef CPU = A->getValue();
 
 #ifdef __wasm__
     // Handle "native" by examining the host. "native" isn't meaningful when
     // cross compiling, so only support this when the host is also WebAssembly.
     if (CPU == "native")
       return llvm::sys::getHostCPUName();
 #endif
 
     return CPU;
   }
 
   return "generic";
 }
 
 std::string tools::getCPUName(const Driver &D, const ArgList &Args,
                               const llvm::Triple &T, bool FromAs) {
   Arg *A;
 
   switch (T.getArch()) {
   default:
     return "";
 
   case llvm::Triple::aarch64:
   case llvm::Triple::aarch64_32:
   case llvm::Triple::aarch64_be:
     return aarch64::getAArch64TargetCPU(Args, T, A);
 
   case llvm::Triple::arm:
   case llvm::Triple::armeb:
   case llvm::Triple::thumb:
   case llvm::Triple::thumbeb: {
     StringRef MArch, MCPU;
     arm::getARMArchCPUFromArgs(Args, MArch, MCPU, FromAs);
     return arm::getARMTargetCPU(MCPU, MArch, T);
   }
 
   case llvm::Triple::avr:
     if (const Arg *A = Args.getLastArg(options::OPT_mmcu_EQ))
       return A->getValue();
     return "";
 
   case llvm::Triple::m68k:
     return m68k::getM68kTargetCPU(Args);
 
   case llvm::Triple::mips:
   case llvm::Triple::mipsel:
   case llvm::Triple::mips64:
   case llvm::Triple::mips64el: {
     StringRef CPUName;
     StringRef ABIName;
     mips::getMipsCPUAndABI(Args, T, CPUName, ABIName);
     return std::string(CPUName);
   }
 
   case llvm::Triple::nvptx:
   case llvm::Triple::nvptx64:
     if (const Arg *A = Args.getLastArg(options::OPT_march_EQ))
       return A->getValue();
     return "";
 
   case llvm::Triple::ppc:
   case llvm::Triple::ppcle:
   case llvm::Triple::ppc64:
   case llvm::Triple::ppc64le: {
     std::string TargetCPUName = ppc::getPPCTargetCPU(Args);
     // LLVM may default to generating code for the native CPU,
     // but, like gcc, we default to a more generic option for
     // each architecture. (except on AIX)
     if (!TargetCPUName.empty())
       return TargetCPUName;
 
     if (T.isOSAIX())
       TargetCPUName = "pwr7";
     else if (T.getArch() == llvm::Triple::ppc64le)
       TargetCPUName = "ppc64le";
     else if (T.getArch() == llvm::Triple::ppc64)
       TargetCPUName = "ppc64";
     else
       TargetCPUName = "ppc";
 
     return TargetCPUName;
   }
   case llvm::Triple::riscv32:
   case llvm::Triple::riscv64:
     if (const Arg *A = Args.getLastArg(options::OPT_mcpu_EQ))
       return A->getValue();
     return "";
 
   case llvm::Triple::bpfel:
   case llvm::Triple::bpfeb:
   case llvm::Triple::sparc:
   case llvm::Triple::sparcel:
   case llvm::Triple::sparcv9:
     if (const Arg *A = Args.getLastArg(options::OPT_mcpu_EQ))
       return A->getValue();
     if (T.getArch() == llvm::Triple::sparc && T.isOSSolaris())
       return "v9";
     return "";
 
   case llvm::Triple::x86:
   case llvm::Triple::x86_64:
     return x86::getX86TargetCPU(D, Args, T);
 
   case llvm::Triple::hexagon:
     return "hexagon" +
            toolchains::HexagonToolChain::GetTargetCPUVersion(Args).str();
 
   case llvm::Triple::lanai:
     return getLanaiTargetCPU(Args);
 
   case llvm::Triple::systemz:
     return systemz::getSystemZTargetCPU(Args);
 
   case llvm::Triple::r600:
   case llvm::Triple::amdgcn:
     return getAMDGPUTargetGPU(T, Args);
 
   case llvm::Triple::wasm32:
   case llvm::Triple::wasm64:
     return std::string(getWebAssemblyTargetCPU(Args));
   }
 }
 
 llvm::StringRef tools::getLTOParallelism(const ArgList &Args, const Driver &D) {
   Arg *LtoJobsArg = Args.getLastArg(options::OPT_flto_jobs_EQ);
   if (!LtoJobsArg)
     return {};
   if (!llvm::get_threadpool_strategy(LtoJobsArg->getValue()))
     D.Diag(diag::err_drv_invalid_int_value)
         << LtoJobsArg->getAsString(Args) << LtoJobsArg->getValue();
   return LtoJobsArg->getValue();
 }
 
 // CloudABI uses -ffunction-sections and -fdata-sections by default.
 bool tools::isUseSeparateSections(const llvm::Triple &Triple) {
   return Triple.getOS() == llvm::Triple::CloudABI;
 }
 
 void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
                           ArgStringList &CmdArgs, const InputInfo &Output,
                           const InputInfo &Input, bool IsThinLTO) {
   const char *Linker = Args.MakeArgString(ToolChain.GetLinkerPath());
   const Driver &D = ToolChain.getDriver();
   if (llvm::sys::path::filename(Linker) != "ld.lld" &&
       llvm::sys::path::stem(Linker) != "ld.lld") {
     // Tell the linker to load the plugin. This has to come before
     // AddLinkerInputs as gold requires -plugin to come before any -plugin-opt
     // that -Wl might forward.
     CmdArgs.push_back("-plugin");
 
 #if defined(_WIN32)
     const char *Suffix = ".dll";
 #elif defined(__APPLE__)
     const char *Suffix = ".dylib";
 #else
     const char *Suffix = ".so";
 #endif
 
     SmallString<1024> Plugin;
     llvm::sys::path::native(
         Twine(D.Dir) + "/../lib" CLANG_LIBDIR_SUFFIX "/LLVMgold" + Suffix,
         Plugin);
     CmdArgs.push_back(Args.MakeArgString(Plugin));
   }
 
   // Try to pass driver level flags relevant to LTO code generation down to
   // the plugin.
 
   // Handle flags for selecting CPU variants.
   std::string CPU = getCPUName(D, Args, ToolChain.getTriple());
   if (!CPU.empty())
     CmdArgs.push_back(Args.MakeArgString(Twine("-plugin-opt=mcpu=") + CPU));
 
   if (Arg *A = Args.getLastArg(options::OPT_O_Group)) {
     // The optimization level matches
     // CompilerInvocation.cpp:getOptimizationLevel().
     StringRef OOpt;
     if (A->getOption().matches(options::OPT_O4) ||
         A->getOption().matches(options::OPT_Ofast))
       OOpt = "3";
     else if (A->getOption().matches(options::OPT_O)) {
       OOpt = A->getValue();
       if (OOpt == "g")
         OOpt = "1";
       else if (OOpt == "s" || OOpt == "z")
         OOpt = "2";
     } else if (A->getOption().matches(options::OPT_O0))
       OOpt = "0";
     if (!OOpt.empty())
       CmdArgs.push_back(Args.MakeArgString(Twine("-plugin-opt=O") + OOpt));
   }
 
   if (Args.hasArg(options::OPT_gsplit_dwarf)) {
     CmdArgs.push_back(
         Args.MakeArgString(Twine("-plugin-opt=dwo_dir=") +
             Output.getFilename() + "_dwo"));
   }
 
   if (IsThinLTO)
     CmdArgs.push_back("-plugin-opt=thinlto");
 
   StringRef Parallelism = getLTOParallelism(Args, D);
   if (!Parallelism.empty())
     CmdArgs.push_back(
         Args.MakeArgString("-plugin-opt=jobs=" + Twine(Parallelism)));
 
   // If an explicit debugger tuning argument appeared, pass it along.
   if (Arg *A = Args.getLastArg(options::OPT_gTune_Group,
                                options::OPT_ggdbN_Group)) {
     if (A->getOption().matches(options::OPT_glldb))
       CmdArgs.push_back("-plugin-opt=-debugger-tune=lldb");
     else if (A->getOption().matches(options::OPT_gsce))
       CmdArgs.push_back("-plugin-opt=-debugger-tune=sce");
     else if (A->getOption().matches(options::OPT_gdbx))
       CmdArgs.push_back("-plugin-opt=-debugger-tune=dbx");
     else
       CmdArgs.push_back("-plugin-opt=-debugger-tune=gdb");
   }
 
   bool UseSeparateSections =
       isUseSeparateSections(ToolChain.getEffectiveTriple());
 
   if (Args.hasFlag(options::OPT_ffunction_sections,
                    options::OPT_fno_function_sections, UseSeparateSections)) {
     CmdArgs.push_back("-plugin-opt=-function-sections");
   }
 
   if (Args.hasFlag(options::OPT_fdata_sections, options::OPT_fno_data_sections,
                    UseSeparateSections)) {
     CmdArgs.push_back("-plugin-opt=-data-sections");
   }
 
   if (Arg *A = getLastProfileSampleUseArg(Args)) {
     StringRef FName = A->getValue();
     if (!llvm::sys::fs::exists(FName))
       D.Diag(diag::err_drv_no_such_file) << FName;
     else
       CmdArgs.push_back(
           Args.MakeArgString(Twine("-plugin-opt=sample-profile=") + FName));
   }
 
   auto *CSPGOGenerateArg = Args.getLastArg(options::OPT_fcs_profile_generate,
                                            options::OPT_fcs_profile_generate_EQ,
                                            options::OPT_fno_profile_generate);
   if (CSPGOGenerateArg &&
       CSPGOGenerateArg->getOption().matches(options::OPT_fno_profile_generate))
     CSPGOGenerateArg = nullptr;
 
   auto *ProfileUseArg = getLastProfileUseArg(Args);
 
   if (CSPGOGenerateArg) {
     CmdArgs.push_back(Args.MakeArgString("-plugin-opt=cs-profile-generate"));
     if (CSPGOGenerateArg->getOption().matches(
             options::OPT_fcs_profile_generate_EQ)) {
       SmallString<128> Path(CSPGOGenerateArg->getValue());
       llvm::sys::path::append(Path, "default_%m.profraw");
       CmdArgs.push_back(
           Args.MakeArgString(Twine("-plugin-opt=cs-profile-path=") + Path));
     } else
       CmdArgs.push_back(
           Args.MakeArgString("-plugin-opt=cs-profile-path=default_%m.profraw"));
   } else if (ProfileUseArg) {
     SmallString<128> Path(
         ProfileUseArg->getNumValues() == 0 ? "" : ProfileUseArg->getValue());
     if (Path.empty() || llvm::sys::fs::is_directory(Path))
       llvm::sys::path::append(Path, "default.profdata");
     CmdArgs.push_back(Args.MakeArgString(Twine("-plugin-opt=cs-profile-path=") +
                                          Path));
   }
 
   // Pass an option to enable/disable the new pass manager.
   if (auto *A = Args.getLastArg(options::OPT_flegacy_pass_manager,
                                 options::OPT_fno_legacy_pass_manager)) {
     if (A->getOption().matches(options::OPT_flegacy_pass_manager))
       CmdArgs.push_back("-plugin-opt=legacy-pass-manager");
     else
       CmdArgs.push_back("-plugin-opt=new-pass-manager");
   }
 
   // Setup statistics file output.
   SmallString<128> StatsFile = getStatsFileName(Args, Output, Input, D);
   if (!StatsFile.empty())
     CmdArgs.push_back(
         Args.MakeArgString(Twine("-plugin-opt=stats-file=") + StatsFile));
 
   addX86AlignBranchArgs(D, Args, CmdArgs, /*IsLTO=*/true);
 
   // Handle remark diagnostics on screen options: '-Rpass-*'.
   renderRpassOptions(Args, CmdArgs);
 
   // Handle serialized remarks options: '-fsave-optimization-record'
   // and '-foptimization-record-*'.
   if (willEmitRemarks(Args))
     renderRemarksOptions(Args, CmdArgs, ToolChain.getEffectiveTriple(), Input,
                          Output);
 
   // Handle remarks hotness/threshold related options.
   renderRemarksHotnessOptions(Args, CmdArgs);
 
   addMachineOutlinerArgs(D, Args, CmdArgs, ToolChain.getEffectiveTriple(),
                          /*IsLTO=*/true);
 }
 
 void tools::addOpenMPRuntimeSpecificRPath(const ToolChain &TC,
                                           const ArgList &Args,
                                           ArgStringList &CmdArgs) {
 
   if (Args.hasFlag(options::OPT_fopenmp_implicit_rpath,
                    options::OPT_fno_openmp_implicit_rpath, true)) {
     // Default to clang lib / lib64 folder, i.e. the same location as device
     // runtime
     SmallString<256> DefaultLibPath =
         llvm::sys::path::parent_path(TC.getDriver().Dir);
     llvm::sys::path::append(DefaultLibPath, Twine("lib") + CLANG_LIBDIR_SUFFIX);
     CmdArgs.push_back("-rpath");
     CmdArgs.push_back(Args.MakeArgString(DefaultLibPath));
   }
 }
 
+void tools::addOpenMPRuntimeLibraryPath(const ToolChain &TC,
+                                        const ArgList &Args,
+                                        ArgStringList &CmdArgs) {
+  // Default to clang lib / lib64 folder, i.e. the same location as device
+  // runtime.
+  SmallString<256> DefaultLibPath =
+      llvm::sys::path::parent_path(TC.getDriver().Dir);
+  llvm::sys::path::append(DefaultLibPath, Twine("lib") + CLANG_LIBDIR_SUFFIX);
+  CmdArgs.push_back(Args.MakeArgString("-L" + DefaultLibPath));
+}
+
 void tools::addArchSpecificRPath(const ToolChain &TC, const ArgList &Args,
                                  ArgStringList &CmdArgs) {
   // Enable -frtlib-add-rpath by default for the case of VE.
   const bool IsVE = TC.getTriple().isVE();
   bool DefaultValue = IsVE;
   if (!Args.hasFlag(options::OPT_frtlib_add_rpath,
                     options::OPT_fno_rtlib_add_rpath, DefaultValue))
     return;
 
   std::string CandidateRPath = TC.getArchSpecificLibPath();
   if (TC.getVFS().exists(CandidateRPath)) {
     CmdArgs.push_back("-rpath");
     CmdArgs.push_back(Args.MakeArgString(CandidateRPath));
   }
 }
 
 bool tools::addOpenMPRuntime(ArgStringList &CmdArgs, const ToolChain &TC,
                              const ArgList &Args, bool ForceStaticHostRuntime,
                              bool IsOffloadingHost, bool GompNeedsRT) {
   if (!Args.hasFlag(options::OPT_fopenmp, options::OPT_fopenmp_EQ,
                     options::OPT_fno_openmp, false))
     return false;
 
   Driver::OpenMPRuntimeKind RTKind = TC.getDriver().getOpenMPRuntime(Args);
 
   if (RTKind == Driver::OMPRT_Unknown)
     // Already diagnosed.
     return false;
 
   if (ForceStaticHostRuntime)
     CmdArgs.push_back("-Bstatic");
 
   switch (RTKind) {
   case Driver::OMPRT_OMP:
     CmdArgs.push_back("-lomp");
     break;
   case Driver::OMPRT_GOMP:
     CmdArgs.push_back("-lgomp");
     break;
   case Driver::OMPRT_IOMP5:
     CmdArgs.push_back("-liomp5");
     break;
   case Driver::OMPRT_Unknown:
     break;
   }
 
   if (ForceStaticHostRuntime)
     CmdArgs.push_back("-Bdynamic");
 
   if (RTKind == Driver::OMPRT_GOMP && GompNeedsRT)
       CmdArgs.push_back("-lrt");
 
   if (IsOffloadingHost)
     CmdArgs.push_back("-lomptarget");
 
   addArchSpecificRPath(TC, Args, CmdArgs);
 
   if (RTKind == Driver::OMPRT_OMP)
     addOpenMPRuntimeSpecificRPath(TC, Args, CmdArgs);
+  addOpenMPRuntimeLibraryPath(TC, Args, CmdArgs);
 
   return true;
 }
 
 static void addSanitizerRuntime(const ToolChain &TC, const ArgList &Args,
                                 ArgStringList &CmdArgs, StringRef Sanitizer,
                                 bool IsShared, bool IsWhole) {
   // Wrap any static runtimes that must be forced into executable in
   // whole-archive.
   if (IsWhole) CmdArgs.push_back("--whole-archive");
   CmdArgs.push_back(TC.getCompilerRTArgString(
       Args, Sanitizer, IsShared ? ToolChain::FT_Shared : ToolChain::FT_Static));
   if (IsWhole) CmdArgs.push_back("--no-whole-archive");
 
   if (IsShared) {
     addArchSpecificRPath(TC, Args, CmdArgs);
   }
 }
 
 // Tries to use a file with the list of dynamic symbols that need to be exported
 // from the runtime library. Returns true if the file was found.
 static bool addSanitizerDynamicList(const ToolChain &TC, const ArgList &Args,
                                     ArgStringList &CmdArgs,
                                     StringRef Sanitizer) {
   // Solaris ld defaults to --export-dynamic behaviour but doesn't support
   // the option, so don't try to pass it.
   if (TC.getTriple().getOS() == llvm::Triple::Solaris)
     return true;
   SmallString<128> SanRT(TC.getCompilerRT(Args, Sanitizer));
   if (llvm::sys::fs::exists(SanRT + ".syms")) {
     CmdArgs.push_back(Args.MakeArgString("--dynamic-list=" + SanRT + ".syms"));
     return true;
   }
   return false;
 }
 
 const char *tools::getAsNeededOption(const ToolChain &TC, bool as_needed) {
   assert(!TC.getTriple().isOSAIX() &&
          "AIX linker does not support any form of --as-needed option yet.");
 
   // While the Solaris 11.2 ld added --as-needed/--no-as-needed as aliases
   // for the native forms -z ignore/-z record, they are missing in Illumos,
   // so always use the native form.
   if (TC.getTriple().isOSSolaris())
     return as_needed ? "-zignore" : "-zrecord";
   else
     return as_needed ? "--as-needed" : "--no-as-needed";
 }
 
 void tools::linkSanitizerRuntimeDeps(const ToolChain &TC,
                                      ArgStringList &CmdArgs) {
   // Fuchsia never needs these.  Any sanitizer runtimes with system
   // dependencies use the `.deplibs` feature instead.
   if (TC.getTriple().isOSFuchsia())
     return;
 
   // Force linking against the system libraries sanitizers depends on
   // (see PR15823 why this is necessary).
   CmdArgs.push_back(getAsNeededOption(TC, false));
   // There's no libpthread or librt on RTEMS & Android.
   if (TC.getTriple().getOS() != llvm::Triple::RTEMS &&
       !TC.getTriple().isAndroid()) {
     CmdArgs.push_back("-lpthread");
     if (!TC.getTriple().isOSOpenBSD())
       CmdArgs.push_back("-lrt");
   }
   CmdArgs.push_back("-lm");
   // There's no libdl on all OSes.
   if (!TC.getTriple().isOSFreeBSD() && !TC.getTriple().isOSNetBSD() &&
       !TC.getTriple().isOSOpenBSD() &&
       TC.getTriple().getOS() != llvm::Triple::RTEMS)
     CmdArgs.push_back("-ldl");
   // Required for backtrace on some OSes
   if (TC.getTriple().isOSFreeBSD() ||
       TC.getTriple().isOSNetBSD() ||
       TC.getTriple().isOSOpenBSD())
     CmdArgs.push_back("-lexecinfo");
 }
 
 static void
 collectSanitizerRuntimes(const ToolChain &TC, const ArgList &Args,
                          SmallVectorImpl<StringRef> &SharedRuntimes,
                          SmallVectorImpl<StringRef> &StaticRuntimes,
                          SmallVectorImpl<StringRef> &NonWholeStaticRuntimes,
                          SmallVectorImpl<StringRef> &HelperStaticRuntimes,
                          SmallVectorImpl<StringRef> &RequiredSymbols) {
   const SanitizerArgs &SanArgs = TC.getSanitizerArgs(Args);
   // Collect shared runtimes.
   if (SanArgs.needsSharedRt()) {
     if (SanArgs.needsAsanRt() && SanArgs.linkRuntimes()) {
       SharedRuntimes.push_back("asan");
       if (!Args.hasArg(options::OPT_shared) && !TC.getTriple().isAndroid())
         HelperStaticRuntimes.push_back("asan-preinit");
     }
     if (SanArgs.needsMemProfRt() && SanArgs.linkRuntimes()) {
       SharedRuntimes.push_back("memprof");
       if (!Args.hasArg(options::OPT_shared) && !TC.getTriple().isAndroid())
         HelperStaticRuntimes.push_back("memprof-preinit");
     }
     if (SanArgs.needsUbsanRt() && SanArgs.linkRuntimes()) {
       if (SanArgs.requiresMinimalRuntime())
         SharedRuntimes.push_back("ubsan_minimal");
       else
         SharedRuntimes.push_back("ubsan_standalone");
     }
     if (SanArgs.needsScudoRt() && SanArgs.linkRuntimes()) {
       if (SanArgs.requiresMinimalRuntime())
         SharedRuntimes.push_back("scudo_minimal");
       else
         SharedRuntimes.push_back("scudo");
     }
     if (SanArgs.needsTsanRt() && SanArgs.linkRuntimes())
       SharedRuntimes.push_back("tsan");
     if (SanArgs.needsHwasanRt() && SanArgs.linkRuntimes()) {
       if (SanArgs.needsHwasanAliasesRt())
         SharedRuntimes.push_back("hwasan_aliases");
       else
         SharedRuntimes.push_back("hwasan");
     }
   }
 
   // The stats_client library is also statically linked into DSOs.
   if (SanArgs.needsStatsRt() && SanArgs.linkRuntimes())
     StaticRuntimes.push_back("stats_client");
 
   // Always link the static runtime regardless of DSO or executable.
   if (SanArgs.needsAsanRt())
     HelperStaticRuntimes.push_back("asan_static");
 
   // Collect static runtimes.
   if (Args.hasArg(options::OPT_shared)) {
     // Don't link static runtimes into DSOs.
     return;
   }
 
   // Each static runtime that has a DSO counterpart above is excluded below,
   // but runtimes that exist only as static are not affected by needsSharedRt.
 
   if (!SanArgs.needsSharedRt() && SanArgs.needsAsanRt() && SanArgs.linkRuntimes()) {
     StaticRuntimes.push_back("asan");
     if (SanArgs.linkCXXRuntimes())
       StaticRuntimes.push_back("asan_cxx");
   }
 
   if (!SanArgs.needsSharedRt() && SanArgs.needsMemProfRt() &&
       SanArgs.linkRuntimes()) {
     StaticRuntimes.push_back("memprof");
     if (SanArgs.linkCXXRuntimes())
       StaticRuntimes.push_back("memprof_cxx");
   }
 
   if (!SanArgs.needsSharedRt() && SanArgs.needsHwasanRt() && SanArgs.linkRuntimes()) {
     if (SanArgs.needsHwasanAliasesRt()) {
       StaticRuntimes.push_back("hwasan_aliases");
       if (SanArgs.linkCXXRuntimes())
         StaticRuntimes.push_back("hwasan_aliases_cxx");
     } else {
       StaticRuntimes.push_back("hwasan");
       if (SanArgs.linkCXXRuntimes())
         StaticRuntimes.push_back("hwasan_cxx");
     }
   }
   if (SanArgs.needsDfsanRt() && SanArgs.linkRuntimes())
     StaticRuntimes.push_back("dfsan");
   if (SanArgs.needsLsanRt() && SanArgs.linkRuntimes())
     StaticRuntimes.push_back("lsan");
   if (SanArgs.needsMsanRt() && SanArgs.linkRuntimes()) {
     StaticRuntimes.push_back("msan");
     if (SanArgs.linkCXXRuntimes())
       StaticRuntimes.push_back("msan_cxx");
   }
   if (!SanArgs.needsSharedRt() && SanArgs.needsTsanRt() &&
       SanArgs.linkRuntimes()) {
     StaticRuntimes.push_back("tsan");
     if (SanArgs.linkCXXRuntimes())
       StaticRuntimes.push_back("tsan_cxx");
   }
   if (!SanArgs.needsSharedRt() && SanArgs.needsUbsanRt() && SanArgs.linkRuntimes()) {
     if (SanArgs.requiresMinimalRuntime()) {
       StaticRuntimes.push_back("ubsan_minimal");
     } else {
       StaticRuntimes.push_back("ubsan_standalone");
       if (SanArgs.linkCXXRuntimes())
         StaticRuntimes.push_back("ubsan_standalone_cxx");
     }
   }
   if (SanArgs.needsSafeStackRt() && SanArgs.linkRuntimes()) {
     NonWholeStaticRuntimes.push_back("safestack");
     RequiredSymbols.push_back("__safestack_init");
   }
   if (!(SanArgs.needsSharedRt() && SanArgs.needsUbsanRt() && SanArgs.linkRuntimes())) {
     if (SanArgs.needsCfiRt() && SanArgs.linkRuntimes())
       StaticRuntimes.push_back("cfi");
     if (SanArgs.needsCfiDiagRt() && SanArgs.linkRuntimes()) {
       StaticRuntimes.push_back("cfi_diag");
       if (SanArgs.linkCXXRuntimes())
         StaticRuntimes.push_back("ubsan_standalone_cxx");
     }
   }
   if (SanArgs.needsStatsRt() && SanArgs.linkRuntimes()) {
     NonWholeStaticRuntimes.push_back("stats");
     RequiredSymbols.push_back("__sanitizer_stats_register");
   }
   if (!SanArgs.needsSharedRt() && SanArgs.needsScudoRt() && SanArgs.linkRuntimes()) {
     if (SanArgs.requiresMinimalRuntime()) {
       StaticRuntimes.push_back("scudo_minimal");
       if (SanArgs.linkCXXRuntimes())
         StaticRuntimes.push_back("scudo_cxx_minimal");
     } else {
       StaticRuntimes.push_back("scudo");
       if (SanArgs.linkCXXRuntimes())
         StaticRuntimes.push_back("scudo_cxx");
     }
   }
 }
 
 // Should be called before we add system libraries (C++ ABI, libstdc++/libc++,
 // C runtime, etc). Returns true if sanitizer system deps need to be linked in.
 bool tools::addSanitizerRuntimes(const ToolChain &TC, const ArgList &Args,
                                  ArgStringList &CmdArgs) {
   SmallVector<StringRef, 4> SharedRuntimes, StaticRuntimes,
       NonWholeStaticRuntimes, HelperStaticRuntimes, RequiredSymbols;
   collectSanitizerRuntimes(TC, Args, SharedRuntimes, StaticRuntimes,
                            NonWholeStaticRuntimes, HelperStaticRuntimes,
                            RequiredSymbols);
 
   const SanitizerArgs &SanArgs = TC.getSanitizerArgs(Args);
   // Inject libfuzzer dependencies.
   if (SanArgs.needsFuzzer() && SanArgs.linkRuntimes() &&
       !Args.hasArg(options::OPT_shared)) {
 
     addSanitizerRuntime(TC, Args, CmdArgs, "fuzzer", false, true);
     if (SanArgs.needsFuzzerInterceptors())
       addSanitizerRuntime(TC, Args, CmdArgs, "fuzzer_interceptors", false,
                           true);
     if (!Args.hasArg(clang::driver::options::OPT_nostdlibxx)) {
       bool OnlyLibstdcxxStatic = Args.hasArg(options::OPT_static_libstdcxx) &&
                                  !Args.hasArg(options::OPT_static);
       if (OnlyLibstdcxxStatic)
         CmdArgs.push_back("-Bstatic");
       TC.AddCXXStdlibLibArgs(Args, CmdArgs);
       if (OnlyLibstdcxxStatic)
         CmdArgs.push_back("-Bdynamic");
     }
   }
 
   for (auto RT : SharedRuntimes)
     addSanitizerRuntime(TC, Args, CmdArgs, RT, true, false);
   for (auto RT : HelperStaticRuntimes)
     addSanitizerRuntime(TC, Args, CmdArgs, RT, false, true);
   bool AddExportDynamic = false;
   for (auto RT : StaticRuntimes) {
     addSanitizerRuntime(TC, Args, CmdArgs, RT, false, true);
     AddExportDynamic |= !addSanitizerDynamicList(TC, Args, CmdArgs, RT);
   }
   for (auto RT : NonWholeStaticRuntimes) {
     addSanitizerRuntime(TC, Args, CmdArgs, RT, false, false);
     AddExportDynamic |= !addSanitizerDynamicList(TC, Args, CmdArgs, RT);
   }
   for (auto S : RequiredSymbols) {
     CmdArgs.push_back("-u");
     CmdArgs.push_back(Args.MakeArgString(S));
   }
   // If there is a static runtime with no dynamic list, force all the symbols
   // to be dynamic to be sure we export sanitizer interface functions.
   if (AddExportDynamic)
     CmdArgs.push_back("--export-dynamic");
 
   if (SanArgs.hasCrossDsoCfi() && !AddExportDynamic)
     CmdArgs.push_back("--export-dynamic-symbol=__cfi_check");
 
   return !StaticRuntimes.empty() || !NonWholeStaticRuntimes.empty();
 }
 
 bool tools::addXRayRuntime(const ToolChain&TC, const ArgList &Args, ArgStringList &CmdArgs) {
   if (Args.hasArg(options::OPT_shared))
     return false;
 
   if (TC.getXRayArgs().needsXRayRt()) {
     CmdArgs.push_back("-whole-archive");
     CmdArgs.push_back(TC.getCompilerRTArgString(Args, "xray"));
     for (const auto &Mode : TC.getXRayArgs().modeList())
       CmdArgs.push_back(TC.getCompilerRTArgString(Args, Mode));
     CmdArgs.push_back("-no-whole-archive");
     return true;
   }
 
   return false;
 }
 
 void tools::linkXRayRuntimeDeps(const ToolChain &TC, ArgStringList &CmdArgs) {
   CmdArgs.push_back(getAsNeededOption(TC, false));
   CmdArgs.push_back("-lpthread");
   if (!TC.getTriple().isOSOpenBSD())
     CmdArgs.push_back("-lrt");
   CmdArgs.push_back("-lm");
 
   if (!TC.getTriple().isOSFreeBSD() &&
       !TC.getTriple().isOSNetBSD() &&
       !TC.getTriple().isOSOpenBSD())
     CmdArgs.push_back("-ldl");
 }
 
 bool tools::areOptimizationsEnabled(const ArgList &Args) {
   // Find the last -O arg and see if it is non-zero.
   if (Arg *A = Args.getLastArg(options::OPT_O_Group))
     return !A->getOption().matches(options::OPT_O0);
   // Defaults to -O0.
   return false;
 }
 
 const char *tools::SplitDebugName(const JobAction &JA, const ArgList &Args,
                                   const InputInfo &Input,
                                   const InputInfo &Output) {
   auto AddPostfix = [JA](auto &F) {
     if (JA.getOffloadingDeviceKind() == Action::OFK_HIP)
       F += (Twine("_") + JA.getOffloadingArch()).str();
     F += ".dwo";
   };
   if (Arg *A = Args.getLastArg(options::OPT_gsplit_dwarf_EQ))
     if (StringRef(A->getValue()) == "single")
       return Args.MakeArgString(Output.getFilename());
 
   Arg *FinalOutput = Args.getLastArg(options::OPT_o);
   if (FinalOutput && Args.hasArg(options::OPT_c)) {
     SmallString<128> T(FinalOutput->getValue());
     llvm::sys::path::remove_filename(T);
     llvm::sys::path::append(T, llvm::sys::path::stem(FinalOutput->getValue()));
     AddPostfix(T);
     return Args.MakeArgString(T);
   } else {
     // Use the compilation dir.
     Arg *A = Args.getLastArg(options::OPT_ffile_compilation_dir_EQ,
                              options::OPT_fdebug_compilation_dir_EQ);
     SmallString<128> T(A ? A->getValue() : "");
     SmallString<128> F(llvm::sys::path::stem(Input.getBaseInput()));
     AddPostfix(F);
     T += F;
     return Args.MakeArgString(T);
   }
 }
 
 void tools::SplitDebugInfo(const ToolChain &TC, Compilation &C, const Tool &T,
                            const JobAction &JA, const ArgList &Args,
                            const InputInfo &Output, const char *OutFile) {
   ArgStringList ExtractArgs;
   ExtractArgs.push_back("--extract-dwo");
 
   ArgStringList StripArgs;
   StripArgs.push_back("--strip-dwo");
 
   // Grabbing the output of the earlier compile step.
   StripArgs.push_back(Output.getFilename());
   ExtractArgs.push_back(Output.getFilename());
   ExtractArgs.push_back(OutFile);
 
   const char *Exec =
       Args.MakeArgString(TC.GetProgramPath(CLANG_DEFAULT_OBJCOPY));
   InputInfo II(types::TY_Object, Output.getFilename(), Output.getFilename());
 
   // First extract the dwo sections.
   C.addCommand(std::make_unique<Command>(JA, T,
                                          ResponseFileSupport::AtFileCurCP(),
                                          Exec, ExtractArgs, II, Output));
 
   // Then remove them from the original .o file.
   C.addCommand(std::make_unique<Command>(
       JA, T, ResponseFileSupport::AtFileCurCP(), Exec, StripArgs, II, Output));
 }
 
 // Claim options we don't want to warn if they are unused. We do this for
 // options that build systems might add but are unused when assembling or only
 // running the preprocessor for example.
 void tools::claimNoWarnArgs(const ArgList &Args) {
   // Don't warn about unused -f(no-)?lto.  This can happen when we're
   // preprocessing, precompiling or assembling.
   Args.ClaimAllArgs(options::OPT_flto_EQ);
   Args.ClaimAllArgs(options::OPT_flto);
   Args.ClaimAllArgs(options::OPT_fno_lto);
 }
 
 Arg *tools::getLastProfileUseArg(const ArgList &Args) {
   auto *ProfileUseArg = Args.getLastArg(
       options::OPT_fprofile_instr_use, options::OPT_fprofile_instr_use_EQ,
       options::OPT_fprofile_use, options::OPT_fprofile_use_EQ,
       options::OPT_fno_profile_instr_use);
 
   if (ProfileUseArg &&
       ProfileUseArg->getOption().matches(options::OPT_fno_profile_instr_use))
     ProfileUseArg = nullptr;
 
   return ProfileUseArg;
 }
 
 Arg *tools::getLastProfileSampleUseArg(const ArgList &Args) {
   auto *ProfileSampleUseArg = Args.getLastArg(
       options::OPT_fprofile_sample_use, options::OPT_fprofile_sample_use_EQ,
       options::OPT_fauto_profile, options::OPT_fauto_profile_EQ,
       options::OPT_fno_profile_sample_use, options::OPT_fno_auto_profile);
 
   if (ProfileSampleUseArg &&
       (ProfileSampleUseArg->getOption().matches(
            options::OPT_fno_profile_sample_use) ||
        ProfileSampleUseArg->getOption().matches(options::OPT_fno_auto_profile)))
     return nullptr;
 
   return Args.getLastArg(options::OPT_fprofile_sample_use_EQ,
                          options::OPT_fauto_profile_EQ);
 }
 
 /// Parses the various -fpic/-fPIC/-fpie/-fPIE arguments.  Then,
 /// smooshes them together with platform defaults, to decide whether
 /// this compile should be using PIC mode or not. Returns a tuple of
 /// (RelocationModel, PICLevel, IsPIE).
 std::tuple<llvm::Reloc::Model, unsigned, bool>
 tools::ParsePICArgs(const ToolChain &ToolChain, const ArgList &Args) {
   const llvm::Triple &EffectiveTriple = ToolChain.getEffectiveTriple();
   const llvm::Triple &Triple = ToolChain.getTriple();
 
   bool PIE = ToolChain.isPIEDefault(Args);
   bool PIC = PIE || ToolChain.isPICDefault();
   // The Darwin/MachO default to use PIC does not apply when using -static.
   if (Triple.isOSBinFormatMachO() && Args.hasArg(options::OPT_static))
     PIE = PIC = false;
   bool IsPICLevelTwo = PIC;
 
   bool KernelOrKext =
       Args.hasArg(options::OPT_mkernel, options::OPT_fapple_kext);
 
   // Android-specific defaults for PIC/PIE
   if (Triple.isAndroid()) {
     switch (Triple.getArch()) {
     case llvm::Triple::arm:
     case llvm::Triple::armeb:
     case llvm::Triple::thumb:
     case llvm::Triple::thumbeb:
     case llvm::Triple::aarch64:
     case llvm::Triple::mips:
     case llvm::Triple::mipsel:
     case llvm::Triple::mips64:
     case llvm::Triple::mips64el:
       PIC = true; // "-fpic"
       break;
 
     case llvm::Triple::x86:
     case llvm::Triple::x86_64:
       PIC = true; // "-fPIC"
       IsPICLevelTwo = true;
       break;
 
     default:
       break;
     }
   }
 
   // OpenBSD-specific defaults for PIE
   if (Triple.isOSOpenBSD()) {
     switch (ToolChain.getArch()) {
     case llvm::Triple::arm:
     case llvm::Triple::aarch64:
     case llvm::Triple::mips64:
     case llvm::Triple::mips64el:
     case llvm::Triple::x86:
     case llvm::Triple::x86_64:
       IsPICLevelTwo = false; // "-fpie"
       break;
 
     case llvm::Triple::ppc:
     case llvm::Triple::sparcv9:
       IsPICLevelTwo = true; // "-fPIE"
       break;
 
     default:
       break;
     }
   }
 
   // AMDGPU-specific defaults for PIC.
   if (Triple.getArch() == llvm::Triple::amdgcn)
     PIC = true;
 
   // The last argument relating to either PIC or PIE wins, and no
   // other argument is used. If the last argument is any flavor of the
   // '-fno-...' arguments, both PIC and PIE are disabled. Any PIE
   // option implicitly enables PIC at the same level.
   Arg *LastPICArg = Args.getLastArg(options::OPT_fPIC, options::OPT_fno_PIC,
                                     options::OPT_fpic, options::OPT_fno_pic,
                                     options::OPT_fPIE, options::OPT_fno_PIE,
                                     options::OPT_fpie, options::OPT_fno_pie);
   if (Triple.isOSWindows() && !Triple.isOSCygMing() && LastPICArg &&
       LastPICArg == Args.getLastArg(options::OPT_fPIC, options::OPT_fpic,
                                     options::OPT_fPIE, options::OPT_fpie)) {
     ToolChain.getDriver().Diag(diag::err_drv_unsupported_opt_for_target)
         << LastPICArg->getSpelling() << Triple.str();
     if (Triple.getArch() == llvm::Triple::x86_64)
       return std::make_tuple(llvm::Reloc::PIC_, 2U, false);
     return std::make_tuple(llvm::Reloc::Static, 0U, false);
   }
 
   // Check whether the tool chain trumps the PIC-ness decision. If the PIC-ness
   // is forced, then neither PIC nor PIE flags will have no effect.
   if (!ToolChain.isPICDefaultForced()) {
     if (LastPICArg) {
       Option O = LastPICArg->getOption();
       if (O.matches(options::OPT_fPIC) || O.matches(options::OPT_fpic) ||
           O.matches(options::OPT_fPIE) || O.matches(options::OPT_fpie)) {
         PIE = O.matches(options::OPT_fPIE) || O.matches(options::OPT_fpie);
         PIC =
             PIE || O.matches(options::OPT_fPIC) || O.matches(options::OPT_fpic);
         IsPICLevelTwo =
             O.matches(options::OPT_fPIE) || O.matches(options::OPT_fPIC);
       } else {
         PIE = PIC = false;
         if (EffectiveTriple.isPS4CPU()) {
           Arg *ModelArg = Args.getLastArg(options::OPT_mcmodel_EQ);
           StringRef Model = ModelArg ? ModelArg->getValue() : "";
           if (Model != "kernel") {
             PIC = true;
             ToolChain.getDriver().Diag(diag::warn_drv_ps4_force_pic)
                 << LastPICArg->getSpelling();
           }
         }
       }
     }
   }
 
   // Introduce a Darwin and PS4-specific hack. If the default is PIC, but the
   // PIC level would've been set to level 1, force it back to level 2 PIC
   // instead.
   if (PIC && (Triple.isOSDarwin() || EffectiveTriple.isPS4CPU()))
     IsPICLevelTwo |= ToolChain.isPICDefault();
 
   // This kernel flags are a trump-card: they will disable PIC/PIE
   // generation, independent of the argument order.
   if (KernelOrKext &&
       ((!EffectiveTriple.isiOS() || EffectiveTriple.isOSVersionLT(6)) &&
        !EffectiveTriple.isWatchOS()))
     PIC = PIE = false;
 
   if (Arg *A = Args.getLastArg(options::OPT_mdynamic_no_pic)) {
     // This is a very special mode. It trumps the other modes, almost no one
     // uses it, and it isn't even valid on any OS but Darwin.
     if (!Triple.isOSDarwin())
       ToolChain.getDriver().Diag(diag::err_drv_unsupported_opt_for_target)
           << A->getSpelling() << Triple.str();
 
     // FIXME: Warn when this flag trumps some other PIC or PIE flag.
 
     // Only a forced PIC mode can cause the actual compile to have PIC defines
     // etc., no flags are sufficient. This behavior was selected to closely
     // match that of llvm-gcc and Apple GCC before that.
     PIC = ToolChain.isPICDefault() && ToolChain.isPICDefaultForced();
 
     return std::make_tuple(llvm::Reloc::DynamicNoPIC, PIC ? 2U : 0U, false);
   }
 
   bool EmbeddedPISupported;
   switch (Triple.getArch()) {
     case llvm::Triple::arm:
     case llvm::Triple::armeb:
     case llvm::Triple::thumb:
     case llvm::Triple::thumbeb:
       EmbeddedPISupported = true;
       break;
     default:
       EmbeddedPISupported = false;
       break;
   }
 
   bool ROPI = false, RWPI = false;
   Arg* LastROPIArg = Args.getLastArg(options::OPT_fropi, options::OPT_fno_ropi);
   if (LastROPIArg && LastROPIArg->getOption().matches(options::OPT_fropi)) {
     if (!EmbeddedPISupported)
       ToolChain.getDriver().Diag(diag::err_drv_unsupported_opt_for_target)
           << LastROPIArg->getSpelling() << Triple.str();
     ROPI = true;
   }
   Arg *LastRWPIArg = Args.getLastArg(options::OPT_frwpi, options::OPT_fno_rwpi);
   if (LastRWPIArg && LastRWPIArg->getOption().matches(options::OPT_frwpi)) {
     if (!EmbeddedPISupported)
       ToolChain.getDriver().Diag(diag::err_drv_unsupported_opt_for_target)
           << LastRWPIArg->getSpelling() << Triple.str();
     RWPI = true;
   }
 
   // ROPI and RWPI are not compatible with PIC or PIE.
   if ((ROPI || RWPI) && (PIC || PIE))
     ToolChain.getDriver().Diag(diag::err_drv_ropi_rwpi_incompatible_with_pic);
 
   if (Triple.isMIPS()) {
     StringRef CPUName;
     StringRef ABIName;
     mips::getMipsCPUAndABI(Args, Triple, CPUName, ABIName);
     // When targeting the N64 ABI, PIC is the default, except in the case
     // when the -mno-abicalls option is used. In that case we exit
     // at next check regardless of PIC being set below.
     if (ABIName == "n64")
       PIC = true;
     // When targettng MIPS with -mno-abicalls, it's always static.
     if(Args.hasArg(options::OPT_mno_abicalls))
       return std::make_tuple(llvm::Reloc::Static, 0U, false);
     // Unlike other architectures, MIPS, even with -fPIC/-mxgot/multigot,
     // does not use PIC level 2 for historical reasons.
     IsPICLevelTwo = false;
   }
 
   if (PIC)
     return std::make_tuple(llvm::Reloc::PIC_, IsPICLevelTwo ? 2U : 1U, PIE);
 
   llvm::Reloc::Model RelocM = llvm::Reloc::Static;
   if (ROPI && RWPI)
     RelocM = llvm::Reloc::ROPI_RWPI;
   else if (ROPI)
     RelocM = llvm::Reloc::ROPI;
   else if (RWPI)
     RelocM = llvm::Reloc::RWPI;
 
   return std::make_tuple(RelocM, 0U, false);
 }
 
 // `-falign-functions` indicates that the functions should be aligned to a
 // 16-byte boundary.
 //
 // `-falign-functions=1` is the same as `-fno-align-functions`.
 //
 // The scalar `n` in `-falign-functions=n` must be an integral value between
 // [0, 65536].  If the value is not a power-of-two, it will be rounded up to
 // the nearest power-of-two.
 //
 // If we return `0`, the frontend will default to the backend's preferred
 // alignment.
 //
 // NOTE: icc only allows values between [0, 4096].  icc uses `-falign-functions`
 // to mean `-falign-functions=16`.  GCC defaults to the backend's preferred
 // alignment.  For unaligned functions, we default to the backend's preferred
 // alignment.
 unsigned tools::ParseFunctionAlignment(const ToolChain &TC,
                                        const ArgList &Args) {
   const Arg *A = Args.getLastArg(options::OPT_falign_functions,
                                  options::OPT_falign_functions_EQ,
                                  options::OPT_fno_align_functions);
   if (!A || A->getOption().matches(options::OPT_fno_align_functions))
     return 0;
 
   if (A->getOption().matches(options::OPT_falign_functions))
     return 0;
 
   unsigned Value = 0;
   if (StringRef(A->getValue()).getAsInteger(10, Value) || Value > 65536)
     TC.getDriver().Diag(diag::err_drv_invalid_int_value)
         << A->getAsString(Args) << A->getValue();
   return Value ? llvm::Log2_32_Ceil(std::min(Value, 65536u)) : Value;
 }
 
 unsigned tools::ParseDebugDefaultVersion(const ToolChain &TC,
                                          const ArgList &Args) {
   const Arg *A = Args.getLastArg(options::OPT_fdebug_default_version);
 
   if (!A)
     return 0;
 
   unsigned Value = 0;
   if (StringRef(A->getValue()).getAsInteger(10, Value) || Value > 5 ||
       Value < 2)
     TC.getDriver().Diag(diag::err_drv_invalid_int_value)
         << A->getAsString(Args) << A->getValue();
   return Value;
 }
 
 void tools::AddAssemblerKPIC(const ToolChain &ToolChain, const ArgList &Args,
                              ArgStringList &CmdArgs) {
   llvm::Reloc::Model RelocationModel;
   unsigned PICLevel;
   bool IsPIE;
   std::tie(RelocationModel, PICLevel, IsPIE) = ParsePICArgs(ToolChain, Args);
 
   if (RelocationModel != llvm::Reloc::Static)
     CmdArgs.push_back("-KPIC");
 }
 
 /// Determine whether Objective-C automated reference counting is
 /// enabled.
 bool tools::isObjCAutoRefCount(const ArgList &Args) {
   return Args.hasFlag(options::OPT_fobjc_arc, options::OPT_fno_objc_arc, false);
 }
 
 enum class LibGccType { UnspecifiedLibGcc, StaticLibGcc, SharedLibGcc };
 
 static LibGccType getLibGccType(const ToolChain &TC, const Driver &D,
                                 const ArgList &Args) {
   if (Args.hasArg(options::OPT_static_libgcc) ||
       Args.hasArg(options::OPT_static) || Args.hasArg(options::OPT_static_pie))
     return LibGccType::StaticLibGcc;
   if (Args.hasArg(options::OPT_shared_libgcc))
     return LibGccType::SharedLibGcc;
   // The Android NDK only provides libunwind.a, not libunwind.so.
   if (TC.getTriple().isAndroid())
     return LibGccType::StaticLibGcc;
   // For MinGW, don't imply a shared libgcc here, we only want to return
   // SharedLibGcc if that was explicitly requested.
   if (D.CCCIsCXX() && !TC.getTriple().isOSCygMing())
     return LibGccType::SharedLibGcc;
   return LibGccType::UnspecifiedLibGcc;
 }
 
 // Gcc adds libgcc arguments in various ways:
 //
 // gcc <none>:     -lgcc --as-needed -lgcc_s --no-as-needed
 // g++ <none>:                       -lgcc_s               -lgcc
 // gcc shared:                       -lgcc_s               -lgcc
 // g++ shared:                       -lgcc_s               -lgcc
 // gcc static:     -lgcc             -lgcc_eh
 // g++ static:     -lgcc             -lgcc_eh
 // gcc static-pie: -lgcc             -lgcc_eh
 // g++ static-pie: -lgcc             -lgcc_eh
 //
 // Also, certain targets need additional adjustments.
 
 static void AddUnwindLibrary(const ToolChain &TC, const Driver &D,
                              ArgStringList &CmdArgs, const ArgList &Args) {
   ToolChain::UnwindLibType UNW = TC.GetUnwindLibType(Args);
   // Targets that don't use unwind libraries.
   if ((TC.getTriple().isAndroid() && UNW == ToolChain::UNW_Libgcc) ||
       TC.getTriple().isOSIAMCU() || TC.getTriple().isOSBinFormatWasm() ||
       UNW == ToolChain::UNW_None)
     return;
 
   LibGccType LGT = getLibGccType(TC, D, Args);
   bool AsNeeded = LGT == LibGccType::UnspecifiedLibGcc &&
                   !TC.getTriple().isAndroid() &&
                   !TC.getTriple().isOSCygMing() && !TC.getTriple().isOSAIX();
   if (AsNeeded)
     CmdArgs.push_back(getAsNeededOption(TC, true));
 
   switch (UNW) {
   case ToolChain::UNW_None:
     return;
   case ToolChain::UNW_Libgcc: {
     if (LGT == LibGccType::StaticLibGcc)
       CmdArgs.push_back("-lgcc_eh");
     else
       CmdArgs.push_back("-lgcc_s");
     break;
   }
   case ToolChain::UNW_CompilerRT:
     if (TC.getTriple().isOSAIX()) {
       // AIX only has libunwind as a shared library. So do not pass
       // anything in if -static is specified.
       if (LGT != LibGccType::StaticLibGcc)
         CmdArgs.push_back("-lunwind");
     } else if (LGT == LibGccType::StaticLibGcc) {
       CmdArgs.push_back("-l:libunwind.a");
     } else if (TC.getTriple().isOSCygMing()) {
       if (LGT == LibGccType::SharedLibGcc)
         CmdArgs.push_back("-l:libunwind.dll.a");
       else
         // Let the linker choose between libunwind.dll.a and libunwind.a
         // depending on what's available, and depending on the -static flag
         CmdArgs.push_back("-lunwind");
     } else {
       CmdArgs.push_back("-l:libunwind.so");
     }
     break;
   }
 
   if (AsNeeded)
     CmdArgs.push_back(getAsNeededOption(TC, false));
 }
 
 static void AddLibgcc(const ToolChain &TC, const Driver &D,
                       ArgStringList &CmdArgs, const ArgList &Args) {
   LibGccType LGT = getLibGccType(TC, D, Args);
   if (LGT != LibGccType::SharedLibGcc)
     CmdArgs.push_back("-lgcc");
   AddUnwindLibrary(TC, D, CmdArgs, Args);
   if (LGT == LibGccType::SharedLibGcc)
     CmdArgs.push_back("-lgcc");
 }
 
 void tools::AddRunTimeLibs(const ToolChain &TC, const Driver &D,
                            ArgStringList &CmdArgs, const ArgList &Args) {
   // Make use of compiler-rt if --rtlib option is used
   ToolChain::RuntimeLibType RLT = TC.GetRuntimeLibType(Args);
 
   switch (RLT) {
   case ToolChain::RLT_CompilerRT:
     CmdArgs.push_back(TC.getCompilerRTArgString(Args, "builtins"));
     AddUnwindLibrary(TC, D, CmdArgs, Args);
     break;
   case ToolChain::RLT_Libgcc:
     // Make sure libgcc is not used under MSVC environment by default
     if (TC.getTriple().isKnownWindowsMSVCEnvironment()) {
       // Issue error diagnostic if libgcc is explicitly specified
       // through command line as --rtlib option argument.
       if (Args.hasArg(options::OPT_rtlib_EQ)) {
         TC.getDriver().Diag(diag::err_drv_unsupported_rtlib_for_platform)
             << Args.getLastArg(options::OPT_rtlib_EQ)->getValue() << "MSVC";
       }
     } else
       AddLibgcc(TC, D, CmdArgs, Args);
     break;
   }
 
   // On Android, the unwinder uses dl_iterate_phdr (or one of
   // dl_unwind_find_exidx/__gnu_Unwind_Find_exidx on arm32) from libdl.so. For
   // statically-linked executables, these functions come from libc.a instead.
   if (TC.getTriple().isAndroid() && !Args.hasArg(options::OPT_static) &&
       !Args.hasArg(options::OPT_static_pie))
     CmdArgs.push_back("-ldl");
 }
 
 SmallString<128> tools::getStatsFileName(const llvm::opt::ArgList &Args,
                                          const InputInfo &Output,
                                          const InputInfo &Input,
                                          const Driver &D) {
   const Arg *A = Args.getLastArg(options::OPT_save_stats_EQ);
   if (!A)
     return {};
 
   StringRef SaveStats = A->getValue();
   SmallString<128> StatsFile;
   if (SaveStats == "obj" && Output.isFilename()) {
     StatsFile.assign(Output.getFilename());
     llvm::sys::path::remove_filename(StatsFile);
   } else if (SaveStats != "cwd") {
     D.Diag(diag::err_drv_invalid_value) << A->getAsString(Args) << SaveStats;
     return {};
   }
 
   StringRef BaseName = llvm::sys::path::filename(Input.getBaseInput());
   llvm::sys::path::append(StatsFile, BaseName);
   llvm::sys::path::replace_extension(StatsFile, "stats");
   return StatsFile;
 }
 
 void tools::addMultilibFlag(bool Enabled, const char *const Flag,
                             Multilib::flags_list &Flags) {
   Flags.push_back(std::string(Enabled ? "+" : "-") + Flag);
 }
 
 void tools::addX86AlignBranchArgs(const Driver &D, const ArgList &Args,
                                   ArgStringList &CmdArgs, bool IsLTO) {
   auto addArg = [&, IsLTO](const Twine &Arg) {
     if (IsLTO) {
       CmdArgs.push_back(Args.MakeArgString("-plugin-opt=" + Arg));
     } else {
       CmdArgs.push_back("-mllvm");
       CmdArgs.push_back(Args.MakeArgString(Arg));
     }
   };
 
   if (Args.hasArg(options::OPT_mbranches_within_32B_boundaries)) {
     addArg(Twine("-x86-branches-within-32B-boundaries"));
   }
   if (const Arg *A = Args.getLastArg(options::OPT_malign_branch_boundary_EQ)) {
     StringRef Value = A->getValue();
     unsigned Boundary;
     if (Value.getAsInteger(10, Boundary) || Boundary < 16 ||
         !llvm::isPowerOf2_64(Boundary)) {
       D.Diag(diag::err_drv_invalid_argument_to_option)
           << Value << A->getOption().getName();
     } else {
       addArg("-x86-align-branch-boundary=" + Twine(Boundary));
     }
   }
   if (const Arg *A = Args.getLastArg(options::OPT_malign_branch_EQ)) {
     std::string AlignBranch;
     for (StringRef T : A->getValues()) {
       if (T != "fused" && T != "jcc" && T != "jmp" && T != "call" &&
           T != "ret" && T != "indirect")
         D.Diag(diag::err_drv_invalid_malign_branch_EQ)
             << T << "fused, jcc, jmp, call, ret, indirect";
       if (!AlignBranch.empty())
         AlignBranch += '+';
       AlignBranch += T;
     }
     addArg("-x86-align-branch=" + Twine(AlignBranch));
   }
   if (const Arg *A = Args.getLastArg(options::OPT_mpad_max_prefix_size_EQ)) {
     StringRef Value = A->getValue();
     unsigned PrefixSize;
     if (Value.getAsInteger(10, PrefixSize)) {
       D.Diag(diag::err_drv_invalid_argument_to_option)
           << Value << A->getOption().getName();
     } else {
       addArg("-x86-pad-max-prefix-size=" + Twine(PrefixSize));
     }
   }
 }
 
 /// SDLSearch: Search for Static Device Library
 /// The search for SDL bitcode files is consistent with how static host
 /// libraries are discovered. That is, the -l option triggers a search for
 /// files in a set of directories called the LINKPATH. The host library search
 /// procedure looks for a specific filename in the LINKPATH.  The filename for
 /// a host library is lib<libname>.a or lib<libname>.so. For SDLs, there is an
 /// ordered-set of filenames that are searched. We call this ordered-set of
 /// filenames as SEARCH-ORDER. Since an SDL can either be device-type specific,
 /// architecture specific, or generic across all architectures, a naming
 /// convention and search order is used where the file name embeds the
 /// architecture name <arch-name> (nvptx or amdgcn) and the GPU device type
 /// <device-name> such as sm_30 and gfx906. <device-name> is absent in case of
 /// device-independent SDLs. To reduce congestion in host library directories,
 /// the search first looks for files in the “libdevice” subdirectory. SDLs that
 /// are bc files begin with the prefix “lib”.
 ///
 /// Machine-code SDLs can also be managed as an archive (*.a file). The
 /// convention has been to use the prefix “lib”. To avoid confusion with host
 /// archive libraries, we use prefix "libbc-" for the bitcode SDL archives.
 ///
 bool tools::SDLSearch(const Driver &D, const llvm::opt::ArgList &DriverArgs,
                       llvm::opt::ArgStringList &CC1Args,
                       SmallVector<std::string, 8> LibraryPaths, std::string Lib,
                       StringRef Arch, StringRef Target, bool isBitCodeSDL,
                       bool postClangLink) {
   SmallVector<std::string, 12> SDLs;
 
   std::string LibDeviceLoc = "/libdevice";
   std::string LibBcPrefix = "/libbc-";
   std::string LibPrefix = "/lib";
 
   if (isBitCodeSDL) {
     // SEARCH-ORDER for Bitcode SDLs:
     //       libdevice/libbc-<libname>-<arch-name>-<device-type>.a
     //       libbc-<libname>-<arch-name>-<device-type>.a
     //       libdevice/libbc-<libname>-<arch-name>.a
     //       libbc-<libname>-<arch-name>.a
     //       libdevice/libbc-<libname>.a
     //       libbc-<libname>.a
     //       libdevice/lib<libname>-<arch-name>-<device-type>.bc
     //       lib<libname>-<arch-name>-<device-type>.bc
     //       libdevice/lib<libname>-<arch-name>.bc
     //       lib<libname>-<arch-name>.bc
     //       libdevice/lib<libname>.bc
     //       lib<libname>.bc
 
     for (StringRef Base : {LibBcPrefix, LibPrefix}) {
       const auto *Ext = Base.contains(LibBcPrefix) ? ".a" : ".bc";
 
       for (auto Suffix : {Twine(Lib + "-" + Arch + "-" + Target).str(),
                           Twine(Lib + "-" + Arch).str(), Twine(Lib).str()}) {
         SDLs.push_back(Twine(LibDeviceLoc + Base + Suffix + Ext).str());
         SDLs.push_back(Twine(Base + Suffix + Ext).str());
       }
     }
   } else {
     // SEARCH-ORDER for Machine-code SDLs:
     //    libdevice/lib<libname>-<arch-name>-<device-type>.a
     //    lib<libname>-<arch-name>-<device-type>.a
     //    libdevice/lib<libname>-<arch-name>.a
     //    lib<libname>-<arch-name>.a
 
     const auto *Ext = ".a";
 
     for (auto Suffix : {Twine(Lib + "-" + Arch + "-" + Target).str(),
                         Twine(Lib + "-" + Arch).str()}) {
       SDLs.push_back(Twine(LibDeviceLoc + LibPrefix + Suffix + Ext).str());
       SDLs.push_back(Twine(LibPrefix + Suffix + Ext).str());
     }
   }
 
   // The CUDA toolchain does not use a global device llvm-link before the LLVM
   // backend generates ptx. So currently, the use of bitcode SDL for nvptx is
   // only possible with post-clang-cc1 linking. Clang cc1 has a feature that
   // will link libraries after clang compilation while the LLVM IR is still in
   // memory. This utilizes a clang cc1 option called “-mlink-builtin-bitcode”.
   // This is a clang -cc1 option that is generated by the clang driver. The
   // option value must a full path to an existing file.
   bool FoundSDL = false;
   for (auto LPath : LibraryPaths) {
     for (auto SDL : SDLs) {
       auto FullName = Twine(LPath + SDL).str();
       if (llvm::sys::fs::exists(FullName)) {
         if (postClangLink)
           CC1Args.push_back("-mlink-builtin-bitcode");
         CC1Args.push_back(DriverArgs.MakeArgString(FullName));
         FoundSDL = true;
         break;
       }
     }
     if (FoundSDL)
       break;
   }
   return FoundSDL;
 }
 
 /// Search if a user provided archive file lib<libname>.a exists in any of
 /// the library paths. If so, add a new command to clang-offload-bundler to
 /// unbundle this archive and create a temporary device specific archive. Name
 /// of this SDL is passed to the llvm-link (for amdgcn) or to the
 /// clang-nvlink-wrapper (for nvptx) commands by the driver.
 bool tools::GetSDLFromOffloadArchive(
     Compilation &C, const Driver &D, const Tool &T, const JobAction &JA,
     const InputInfoList &Inputs, const llvm::opt::ArgList &DriverArgs,
     llvm::opt::ArgStringList &CC1Args, SmallVector<std::string, 8> LibraryPaths,
     StringRef Lib, StringRef Arch, StringRef Target, bool isBitCodeSDL,
     bool postClangLink) {
 
   // We don't support bitcode archive bundles for nvptx
   if (isBitCodeSDL && Arch.contains("nvptx"))
     return false;
 
   bool FoundAOB = false;
   SmallVector<std::string, 2> AOBFileNames;
   std::string ArchiveOfBundles;
   for (auto LPath : LibraryPaths) {
     ArchiveOfBundles.clear();
 
     AOBFileNames.push_back(Twine(LPath + "/libdevice/lib" + Lib + ".a").str());
     AOBFileNames.push_back(Twine(LPath + "/lib" + Lib + ".a").str());
 
     for (auto AOB : AOBFileNames) {
       if (llvm::sys::fs::exists(AOB)) {
         ArchiveOfBundles = AOB;
         FoundAOB = true;
         break;
       }
     }
 
     if (!FoundAOB)
       continue;
 
     StringRef Prefix = isBitCodeSDL ? "libbc-" : "lib";
     std::string OutputLib = D.GetTemporaryPath(
         Twine(Prefix + Lib + "-" + Arch + "-" + Target).str(), "a");
 
     C.addTempFile(C.getArgs().MakeArgString(OutputLib));
 
     ArgStringList CmdArgs;
     SmallString<128> DeviceTriple;
     DeviceTriple += Action::GetOffloadKindName(JA.getOffloadingDeviceKind());
     DeviceTriple += '-';
     std::string NormalizedTriple = T.getToolChain().getTriple().normalize();
     DeviceTriple += NormalizedTriple;
     if (!Target.empty()) {
       DeviceTriple += '-';
       DeviceTriple += Target;
     }
 
     std::string UnbundleArg("-unbundle");
     std::string TypeArg("-type=a");
     std::string InputArg("-inputs=" + ArchiveOfBundles);
     std::string OffloadArg("-targets=" + std::string(DeviceTriple));
     std::string OutputArg("-outputs=" + OutputLib);
 
     const char *UBProgram = DriverArgs.MakeArgString(
         T.getToolChain().GetProgramPath("clang-offload-bundler"));
 
     ArgStringList UBArgs;
     UBArgs.push_back(C.getArgs().MakeArgString(UnbundleArg));
     UBArgs.push_back(C.getArgs().MakeArgString(TypeArg));
     UBArgs.push_back(C.getArgs().MakeArgString(InputArg));
     UBArgs.push_back(C.getArgs().MakeArgString(OffloadArg));
     UBArgs.push_back(C.getArgs().MakeArgString(OutputArg));
 
     // Add this flag to not exit from clang-offload-bundler if no compatible
     // code object is found in heterogenous archive library.
     std::string AdditionalArgs("-allow-missing-bundles");
     UBArgs.push_back(C.getArgs().MakeArgString(AdditionalArgs));
 
     C.addCommand(std::make_unique<Command>(
         JA, T, ResponseFileSupport::AtFileCurCP(), UBProgram, UBArgs, Inputs,
         InputInfo(&JA, C.getArgs().MakeArgString(OutputLib))));
     if (postClangLink)
       CC1Args.push_back("-mlink-builtin-bitcode");
 
     CC1Args.push_back(DriverArgs.MakeArgString(OutputLib));
     break;
   }
 
   return FoundAOB;
 }
 
 // Wrapper function used by driver for adding SDLs during link phase.
 void tools::AddStaticDeviceLibsLinking(Compilation &C, const Tool &T,
                                 const JobAction &JA,
                                 const InputInfoList &Inputs,
                                 const llvm::opt::ArgList &DriverArgs,
                                 llvm::opt::ArgStringList &CC1Args,
                                 StringRef Arch, StringRef Target,
                                 bool isBitCodeSDL, bool postClangLink) {
   AddStaticDeviceLibs(&C, &T, &JA, &Inputs, C.getDriver(), DriverArgs, CC1Args,
                       Arch, Target, isBitCodeSDL, postClangLink);
 }
 
 // Wrapper function used for post clang linking of bitcode SDLS for nvptx by
 // the CUDA toolchain.
 void tools::AddStaticDeviceLibsPostLinking(const Driver &D,
                                 const llvm::opt::ArgList &DriverArgs,
                                 llvm::opt::ArgStringList &CC1Args,
                                 StringRef Arch, StringRef Target,
                                 bool isBitCodeSDL, bool postClangLink) {
   AddStaticDeviceLibs(nullptr, nullptr, nullptr, nullptr, D, DriverArgs,
                       CC1Args, Arch, Target, isBitCodeSDL, postClangLink);
 }
 
 // User defined Static Device Libraries(SDLs) can be passed to clang for
 // offloading GPU compilers. Like static host libraries, the use of a SDL is
 // specified with the -l command line option. The primary difference between
 // host and SDLs is the filenames for SDLs (refer SEARCH-ORDER for Bitcode SDLs
 // and SEARCH-ORDER for Machine-code SDLs for the naming convention).
 // SDLs are of following types:
 //
 // * Bitcode SDLs: They can either be a *.bc file or an archive of *.bc files.
 //           For NVPTX, these libraries are post-clang linked following each
 //           compilation. For AMDGPU, these libraries are linked one time
 //           during the application link phase.
 //
 // * Machine-code SDLs: They are archive files. For NVPTX, the archive members
 //           contain cubin for Nvidia GPUs and are linked one time during the
 //           link phase by the CUDA SDK linker called nvlink.	For AMDGPU, the
 //           process for machine code SDLs is still in development. But they
 //           will be linked by the LLVM tool lld.
 //
 // * Bundled objects that contain both host and device codes: Bundled objects
 //           may also contain library code compiled from source. For NVPTX, the
 //           bundle contains cubin. For AMDGPU, the bundle contains bitcode.
 //
 // For Bitcode and Machine-code SDLs, current compiler toolchains hardcode the
 // inclusion of specific SDLs such as math libraries and the OpenMP device
 // library libomptarget.
 void tools::AddStaticDeviceLibs(Compilation *C, const Tool *T,
                                 const JobAction *JA,
                                 const InputInfoList *Inputs, const Driver &D,
                                 const llvm::opt::ArgList &DriverArgs,
                                 llvm::opt::ArgStringList &CC1Args,
                                 StringRef Arch, StringRef Target,
                                 bool isBitCodeSDL, bool postClangLink) {
 
   SmallVector<std::string, 8> LibraryPaths;
   // Add search directories from LIBRARY_PATH env variable
   llvm::Optional<std::string> LibPath =
       llvm::sys::Process::GetEnv("LIBRARY_PATH");
   if (LibPath) {
     SmallVector<StringRef, 8> Frags;
     const char EnvPathSeparatorStr[] = {llvm::sys::EnvPathSeparator, '\0'};
     llvm::SplitString(*LibPath, Frags, EnvPathSeparatorStr);
     for (StringRef Path : Frags)
       LibraryPaths.emplace_back(Path.trim());
   }
 
   // Add directories from user-specified -L options
   for (std::string Search_Dir : DriverArgs.getAllArgValues(options::OPT_L))
     LibraryPaths.emplace_back(Search_Dir);
 
   // Add path to lib-debug folders
   SmallString<256> DefaultLibPath = llvm::sys::path::parent_path(D.Dir);
   llvm::sys::path::append(DefaultLibPath, Twine("lib") + CLANG_LIBDIR_SUFFIX);
   LibraryPaths.emplace_back(DefaultLibPath.c_str());
 
   // Build list of Static Device Libraries SDLs specified by -l option
   llvm::SmallSet<std::string, 16> SDLNames;
   static const StringRef HostOnlyArchives[] = {
       "omp", "cudart", "m", "gcc", "gcc_s", "pthread", "hip_hcc"};
   for (auto SDLName : DriverArgs.getAllArgValues(options::OPT_l)) {
     if (!HostOnlyArchives->contains(SDLName)) {
       SDLNames.insert(SDLName);
     }
   }
 
   // The search stops as soon as an SDL file is found. The driver then provides
   // the full filename of the SDL to the llvm-link or clang-nvlink-wrapper
   // command. If no SDL is found after searching each LINKPATH with
   // SEARCH-ORDER, it is possible that an archive file lib<libname>.a exists
   // and may contain bundled object files.
   for (auto SDLName : SDLNames) {
     // This is the only call to SDLSearch
     if (!SDLSearch(D, DriverArgs, CC1Args, LibraryPaths, SDLName, Arch, Target,
                    isBitCodeSDL, postClangLink)) {
       GetSDLFromOffloadArchive(*C, D, *T, *JA, *Inputs, DriverArgs, CC1Args,
                                LibraryPaths, SDLName, Arch, Target,
                                isBitCodeSDL, postClangLink);
     }
   }
 }
 
 static llvm::opt::Arg *
 getAMDGPUCodeObjectArgument(const Driver &D, const llvm::opt::ArgList &Args) {
   // The last of -mcode-object-v3, -mno-code-object-v3 and
   // -mcode-object-version=<version> wins.
   return Args.getLastArg(options::OPT_mcode_object_v3_legacy,
                          options::OPT_mno_code_object_v3_legacy,
                          options::OPT_mcode_object_version_EQ);
 }
 
 void tools::checkAMDGPUCodeObjectVersion(const Driver &D,
                                          const llvm::opt::ArgList &Args) {
   const unsigned MinCodeObjVer = 2;
   const unsigned MaxCodeObjVer = 4;
 
   // Emit warnings for legacy options even if they are overridden.
   if (Args.hasArg(options::OPT_mno_code_object_v3_legacy))
     D.Diag(diag::warn_drv_deprecated_arg) << "-mno-code-object-v3"
                                           << "-mcode-object-version=2";
 
   if (Args.hasArg(options::OPT_mcode_object_v3_legacy))
     D.Diag(diag::warn_drv_deprecated_arg) << "-mcode-object-v3"
                                           << "-mcode-object-version=3";
 
   if (auto *CodeObjArg = getAMDGPUCodeObjectArgument(D, Args)) {
     if (CodeObjArg->getOption().getID() ==
         options::OPT_mcode_object_version_EQ) {
       unsigned CodeObjVer = MaxCodeObjVer;
       auto Remnant =
           StringRef(CodeObjArg->getValue()).getAsInteger(0, CodeObjVer);
       if (Remnant || CodeObjVer < MinCodeObjVer || CodeObjVer > MaxCodeObjVer)
         D.Diag(diag::err_drv_invalid_int_value)
             << CodeObjArg->getAsString(Args) << CodeObjArg->getValue();
     }
   }
 }
 
 unsigned tools::getAMDGPUCodeObjectVersion(const Driver &D,
                                            const llvm::opt::ArgList &Args) {
   unsigned CodeObjVer = 4; // default
   if (auto *CodeObjArg = getAMDGPUCodeObjectArgument(D, Args)) {
     if (CodeObjArg->getOption().getID() ==
         options::OPT_mno_code_object_v3_legacy) {
       CodeObjVer = 2;
     } else if (CodeObjArg->getOption().getID() ==
                options::OPT_mcode_object_v3_legacy) {
       CodeObjVer = 3;
     } else {
       StringRef(CodeObjArg->getValue()).getAsInteger(0, CodeObjVer);
     }
   }
   return CodeObjVer;
 }
 
 bool tools::haveAMDGPUCodeObjectVersionArgument(
     const Driver &D, const llvm::opt::ArgList &Args) {
   return getAMDGPUCodeObjectArgument(D, Args) != nullptr;
 }
 
 void tools::addMachineOutlinerArgs(const Driver &D,
                                    const llvm::opt::ArgList &Args,
                                    llvm::opt::ArgStringList &CmdArgs,
                                    const llvm::Triple &Triple, bool IsLTO) {
   auto addArg = [&, IsLTO](const Twine &Arg) {
     if (IsLTO) {
       CmdArgs.push_back(Args.MakeArgString("-plugin-opt=" + Arg));
     } else {
       CmdArgs.push_back("-mllvm");
       CmdArgs.push_back(Args.MakeArgString(Arg));
     }
   };
 
   if (Arg *A = Args.getLastArg(options::OPT_moutline,
                                options::OPT_mno_outline)) {
     if (A->getOption().matches(options::OPT_moutline)) {
       // We only support -moutline in AArch64 and ARM targets right now. If
       // we're not compiling for these, emit a warning and ignore the flag.
       // Otherwise, add the proper mllvm flags.
       if (!(Triple.isARM() || Triple.isThumb() ||
             Triple.getArch() == llvm::Triple::aarch64 ||
             Triple.getArch() == llvm::Triple::aarch64_32)) {
         D.Diag(diag::warn_drv_moutline_unsupported_opt) << Triple.getArchName();
       } else {
         addArg(Twine("-enable-machine-outliner"));
       }
     } else {
       // Disable all outlining behaviour.
       addArg(Twine("-enable-machine-outliner=never"));
     }
   }
 }
 
 void tools::addOpenMPDeviceRTL(const Driver &D,
                                const llvm::opt::ArgList &DriverArgs,
                                llvm::opt::ArgStringList &CC1Args,
                                StringRef BitcodeSuffix,
                                const llvm::Triple &Triple) {
   SmallVector<StringRef, 8> LibraryPaths;
 
   // Add path to clang lib / lib64 folder.
   SmallString<256> DefaultLibPath = llvm::sys::path::parent_path(D.Dir);
   llvm::sys::path::append(DefaultLibPath, Twine("lib") + CLANG_LIBDIR_SUFFIX);
   LibraryPaths.emplace_back(DefaultLibPath.c_str());
 
   // Add user defined library paths from LIBRARY_PATH.
   llvm::Optional<std::string> LibPath =
       llvm::sys::Process::GetEnv("LIBRARY_PATH");
   if (LibPath) {
     SmallVector<StringRef, 8> Frags;
     const char EnvPathSeparatorStr[] = {llvm::sys::EnvPathSeparator, '\0'};
     llvm::SplitString(*LibPath, Frags, EnvPathSeparatorStr);
     for (StringRef Path : Frags)
       LibraryPaths.emplace_back(Path.trim());
   }
 
   OptSpecifier LibomptargetBCPathOpt =
       Triple.isAMDGCN() ? options::OPT_libomptarget_amdgcn_bc_path_EQ
                         : options::OPT_libomptarget_nvptx_bc_path_EQ;
 
   StringRef ArchPrefix = Triple.isAMDGCN() ? "amdgcn" : "nvptx";
   std::string LibOmpTargetName = "libomptarget-" + BitcodeSuffix.str() + ".bc";
 
   // First check whether user specifies bc library
   if (const Arg *A = DriverArgs.getLastArg(LibomptargetBCPathOpt)) {
     SmallString<128> LibOmpTargetFile(A->getValue());
     if (llvm::sys::fs::exists(LibOmpTargetFile) &&
         llvm::sys::fs::is_directory(LibOmpTargetFile)) {
       llvm::sys::path::append(LibOmpTargetFile, LibOmpTargetName);
     }
 
     if (llvm::sys::fs::exists(LibOmpTargetFile)) {
       CC1Args.push_back("-mlink-builtin-bitcode");
       CC1Args.push_back(DriverArgs.MakeArgString(LibOmpTargetFile));
     } else {
       D.Diag(diag::err_drv_omp_offload_target_bcruntime_not_found)
           << LibOmpTargetFile;
     }
   } else {
     bool FoundBCLibrary = false;
 
     for (StringRef LibraryPath : LibraryPaths) {
       SmallString<128> LibOmpTargetFile(LibraryPath);
       llvm::sys::path::append(LibOmpTargetFile, LibOmpTargetName);
       if (llvm::sys::fs::exists(LibOmpTargetFile)) {
         CC1Args.push_back("-mlink-builtin-bitcode");
         CC1Args.push_back(DriverArgs.MakeArgString(LibOmpTargetFile));
         FoundBCLibrary = true;
         break;
       }
     }
 
     if (!FoundBCLibrary)
       D.Diag(diag::err_drv_omp_offload_target_missingbcruntime)
           << LibOmpTargetName << ArchPrefix;
   }
 }
diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.h b/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.h
index 23012dc247e4..2bba1ee285e6 100644
--- a/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.h
+++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/CommonArgs.h
@@ -1,199 +1,202 @@
 //===--- CommonArgs.h - Args handling for multiple toolchains ---*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_COMMONARGS_H
 #define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_COMMONARGS_H
 
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/InputInfo.h"
 #include "clang/Driver/Multilib.h"
 #include "clang/Driver/Tool.h"
 #include "clang/Driver/ToolChain.h"
 #include "llvm/Support/CodeGen.h"
 
 namespace clang {
 namespace driver {
 namespace tools {
 
 void addPathIfExists(const Driver &D, const Twine &Path,
                      ToolChain::path_list &Paths);
 
 void AddLinkerInputs(const ToolChain &TC, const InputInfoList &Inputs,
                      const llvm::opt::ArgList &Args,
                      llvm::opt::ArgStringList &CmdArgs, const JobAction &JA);
 
 void addLinkerCompressDebugSectionsOption(const ToolChain &TC,
                                           const llvm::opt::ArgList &Args,
                                           llvm::opt::ArgStringList &CmdArgs);
 
 void claimNoWarnArgs(const llvm::opt::ArgList &Args);
 
 bool addSanitizerRuntimes(const ToolChain &TC, const llvm::opt::ArgList &Args,
                           llvm::opt::ArgStringList &CmdArgs);
 
 void linkSanitizerRuntimeDeps(const ToolChain &TC,
                               llvm::opt::ArgStringList &CmdArgs);
 
 bool addXRayRuntime(const ToolChain &TC, const llvm::opt::ArgList &Args,
                     llvm::opt::ArgStringList &CmdArgs);
 
 void linkXRayRuntimeDeps(const ToolChain &TC,
                          llvm::opt::ArgStringList &CmdArgs);
 
 void AddRunTimeLibs(const ToolChain &TC, const Driver &D,
                     llvm::opt::ArgStringList &CmdArgs,
                     const llvm::opt::ArgList &Args);
 
 void AddStaticDeviceLibsLinking(Compilation &C, const Tool &T,
                                 const JobAction &JA,
                                 const InputInfoList &Inputs,
                                 const llvm::opt::ArgList &DriverArgs,
                                 llvm::opt::ArgStringList &CmdArgs,
                                 StringRef Arch, StringRef Target,
                                 bool isBitCodeSDL, bool postClangLink);
 void AddStaticDeviceLibsPostLinking(const Driver &D,
                                     const llvm::opt::ArgList &DriverArgs,
                                     llvm::opt::ArgStringList &CmdArgs,
                                     StringRef Arch, StringRef Target,
                                     bool isBitCodeSDL, bool postClangLink);
 void AddStaticDeviceLibs(Compilation *C, const Tool *T, const JobAction *JA,
                          const InputInfoList *Inputs, const Driver &D,
                          const llvm::opt::ArgList &DriverArgs,
                          llvm::opt::ArgStringList &CmdArgs, StringRef Arch,
                          StringRef Target, bool isBitCodeSDL,
                          bool postClangLink);
 
 bool SDLSearch(const Driver &D, const llvm::opt::ArgList &DriverArgs,
                llvm::opt::ArgStringList &CmdArgs,
                SmallVector<std::string, 8> LibraryPaths, std::string Lib,
                StringRef Arch, StringRef Target, bool isBitCodeSDL,
                bool postClangLink);
 
 bool GetSDLFromOffloadArchive(Compilation &C, const Driver &D, const Tool &T,
                               const JobAction &JA, const InputInfoList &Inputs,
                               const llvm::opt::ArgList &DriverArgs,
                               llvm::opt::ArgStringList &CC1Args,
                               SmallVector<std::string, 8> LibraryPaths,
                               StringRef Lib, StringRef Arch, StringRef Target,
                               bool isBitCodeSDL, bool postClangLink);
 
 const char *SplitDebugName(const JobAction &JA, const llvm::opt::ArgList &Args,
                            const InputInfo &Input, const InputInfo &Output);
 
 void SplitDebugInfo(const ToolChain &TC, Compilation &C, const Tool &T,
                     const JobAction &JA, const llvm::opt::ArgList &Args,
                     const InputInfo &Output, const char *OutFile);
 
 void addLTOOptions(const ToolChain &ToolChain, const llvm::opt::ArgList &Args,
                    llvm::opt::ArgStringList &CmdArgs, const InputInfo &Output,
                    const InputInfo &Input, bool IsThinLTO);
 
 std::tuple<llvm::Reloc::Model, unsigned, bool>
 ParsePICArgs(const ToolChain &ToolChain, const llvm::opt::ArgList &Args);
 
 unsigned ParseFunctionAlignment(const ToolChain &TC,
                                 const llvm::opt::ArgList &Args);
 
 unsigned ParseDebugDefaultVersion(const ToolChain &TC,
                                   const llvm::opt::ArgList &Args);
 
 void AddAssemblerKPIC(const ToolChain &ToolChain,
                       const llvm::opt::ArgList &Args,
                       llvm::opt::ArgStringList &CmdArgs);
 
 void addOpenMPRuntimeSpecificRPath(const ToolChain &TC,
                                    const llvm::opt::ArgList &Args,
                                    llvm::opt::ArgStringList &CmdArgs);
 void addArchSpecificRPath(const ToolChain &TC, const llvm::opt::ArgList &Args,
                           llvm::opt::ArgStringList &CmdArgs);
+void addOpenMPRuntimeLibraryPath(const ToolChain &TC,
+                                 const llvm::opt::ArgList &Args,
+                                 llvm::opt::ArgStringList &CmdArgs);
 /// Returns true, if an OpenMP runtime has been added.
 bool addOpenMPRuntime(llvm::opt::ArgStringList &CmdArgs, const ToolChain &TC,
                       const llvm::opt::ArgList &Args,
                       bool ForceStaticHostRuntime = false,
                       bool IsOffloadingHost = false, bool GompNeedsRT = false);
 
 const char *getAsNeededOption(const ToolChain &TC, bool as_needed);
 
 llvm::opt::Arg *getLastProfileUseArg(const llvm::opt::ArgList &Args);
 llvm::opt::Arg *getLastProfileSampleUseArg(const llvm::opt::ArgList &Args);
 
 bool isObjCAutoRefCount(const llvm::opt::ArgList &Args);
 
 llvm::StringRef getLTOParallelism(const llvm::opt::ArgList &Args,
                                   const Driver &D);
 
 bool areOptimizationsEnabled(const llvm::opt::ArgList &Args);
 
 bool isUseSeparateSections(const llvm::Triple &Triple);
 
 /// \p EnvVar is split by system delimiter for environment variables.
 /// If \p ArgName is "-I", "-L", or an empty string, each entry from \p EnvVar
 /// is prefixed by \p ArgName then added to \p Args. Otherwise, for each
 /// entry of \p EnvVar, \p ArgName is added to \p Args first, then the entry
 /// itself is added.
 void addDirectoryList(const llvm::opt::ArgList &Args,
                       llvm::opt::ArgStringList &CmdArgs, const char *ArgName,
                       const char *EnvVar);
 
 void AddTargetFeature(const llvm::opt::ArgList &Args,
                       std::vector<StringRef> &Features,
                       llvm::opt::OptSpecifier OnOpt,
                       llvm::opt::OptSpecifier OffOpt, StringRef FeatureName);
 
 std::string getCPUName(const Driver &D, const llvm::opt::ArgList &Args,
                        const llvm::Triple &T, bool FromAs = false);
 
 /// Iterate \p Args and convert -mxxx to +xxx and -mno-xxx to -xxx and
 /// append it to \p Features.
 ///
 /// Note: Since \p Features may contain default values before calling
 /// this function, or may be appended with entries to override arguments,
 /// entries in \p Features are not unique.
 void handleTargetFeaturesGroup(const llvm::opt::ArgList &Args,
                                std::vector<StringRef> &Features,
                                llvm::opt::OptSpecifier Group);
 
 /// If there are multiple +xxx or -xxx features, keep the last one.
 std::vector<StringRef>
 unifyTargetFeatures(const std::vector<StringRef> &Features);
 
 /// Handles the -save-stats option and returns the filename to save statistics
 /// to.
 SmallString<128> getStatsFileName(const llvm::opt::ArgList &Args,
                                   const InputInfo &Output,
                                   const InputInfo &Input, const Driver &D);
 
 /// \p Flag must be a flag accepted by the driver with its leading '-' removed,
 //     otherwise '-print-multi-lib' will not emit them correctly.
 void addMultilibFlag(bool Enabled, const char *const Flag,
                      Multilib::flags_list &Flags);
 
 void addX86AlignBranchArgs(const Driver &D, const llvm::opt::ArgList &Args,
                            llvm::opt::ArgStringList &CmdArgs, bool IsLTO);
 
 void checkAMDGPUCodeObjectVersion(const Driver &D,
                                   const llvm::opt::ArgList &Args);
 
 unsigned getAMDGPUCodeObjectVersion(const Driver &D,
                                     const llvm::opt::ArgList &Args);
 
 bool haveAMDGPUCodeObjectVersionArgument(const Driver &D,
                                          const llvm::opt::ArgList &Args);
 
 void addMachineOutlinerArgs(const Driver &D, const llvm::opt::ArgList &Args,
                             llvm::opt::ArgStringList &CmdArgs,
                             const llvm::Triple &Triple, bool IsLTO);
 
 void addOpenMPDeviceRTL(const Driver &D, const llvm::opt::ArgList &DriverArgs,
                         llvm::opt::ArgStringList &CC1Args,
                         StringRef BitcodeSuffix, const llvm::Triple &Triple);
 } // end namespace tools
 } // end namespace driver
 } // end namespace clang
 
 #endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_COMMONARGS_H
diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/Linux.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/Linux.cpp
index f85c04df4f6c..83cb41159de7 100644
--- a/contrib/llvm-project/clang/lib/Driver/ToolChains/Linux.cpp
+++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/Linux.cpp
@@ -1,774 +1,768 @@
 //===--- Linux.h - Linux ToolChain Implementations --------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "Linux.h"
 #include "Arch/ARM.h"
 #include "Arch/Mips.h"
 #include "Arch/PPC.h"
 #include "Arch/RISCV.h"
 #include "CommonArgs.h"
 #include "clang/Config/config.h"
 #include "clang/Driver/Distro.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Options.h"
 #include "clang/Driver/SanitizerArgs.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include <system_error>
 
 using namespace clang::driver;
 using namespace clang::driver::toolchains;
 using namespace clang;
 using namespace llvm::opt;
 
 using tools::addPathIfExists;
 
 /// Get our best guess at the multiarch triple for a target.
 ///
 /// Debian-based systems are starting to use a multiarch setup where they use
 /// a target-triple directory in the library and header search paths.
 /// Unfortunately, this triple does not align with the vanilla target triple,
 /// so we provide a rough mapping here.
 std::string Linux::getMultiarchTriple(const Driver &D,
                                       const llvm::Triple &TargetTriple,
                                       StringRef SysRoot) const {
   llvm::Triple::EnvironmentType TargetEnvironment =
       TargetTriple.getEnvironment();
   bool IsAndroid = TargetTriple.isAndroid();
   bool IsMipsR6 = TargetTriple.getSubArch() == llvm::Triple::MipsSubArch_r6;
   bool IsMipsN32Abi = TargetTriple.getEnvironment() == llvm::Triple::GNUABIN32;
 
   // For most architectures, just use whatever we have rather than trying to be
   // clever.
   switch (TargetTriple.getArch()) {
   default:
     break;
 
   // We use the existence of '/lib/<triple>' as a directory to detect some
   // common linux triples that don't quite match the Clang triple for both
   // 32-bit and 64-bit targets. Multiarch fixes its install triples to these
   // regardless of what the actual target triple is.
   case llvm::Triple::arm:
   case llvm::Triple::thumb:
     if (IsAndroid)
       return "arm-linux-androideabi";
     if (TargetEnvironment == llvm::Triple::GNUEABIHF)
       return "arm-linux-gnueabihf";
     return "arm-linux-gnueabi";
   case llvm::Triple::armeb:
   case llvm::Triple::thumbeb:
     if (TargetEnvironment == llvm::Triple::GNUEABIHF)
       return "armeb-linux-gnueabihf";
     return "armeb-linux-gnueabi";
   case llvm::Triple::x86:
     if (IsAndroid)
       return "i686-linux-android";
     return "i386-linux-gnu";
   case llvm::Triple::x86_64:
     if (IsAndroid)
       return "x86_64-linux-android";
     if (TargetEnvironment == llvm::Triple::GNUX32)
       return "x86_64-linux-gnux32";
     return "x86_64-linux-gnu";
   case llvm::Triple::aarch64:
     if (IsAndroid)
       return "aarch64-linux-android";
     return "aarch64-linux-gnu";
   case llvm::Triple::aarch64_be:
     return "aarch64_be-linux-gnu";
 
   case llvm::Triple::m68k:
     return "m68k-linux-gnu";
 
   case llvm::Triple::mips:
     return IsMipsR6 ? "mipsisa32r6-linux-gnu" : "mips-linux-gnu";
   case llvm::Triple::mipsel:
     if (IsAndroid)
       return "mipsel-linux-android";
     return IsMipsR6 ? "mipsisa32r6el-linux-gnu" : "mipsel-linux-gnu";
   case llvm::Triple::mips64: {
     std::string MT = std::string(IsMipsR6 ? "mipsisa64r6" : "mips64") +
                      "-linux-" + (IsMipsN32Abi ? "gnuabin32" : "gnuabi64");
     if (D.getVFS().exists(SysRoot + "/lib/" + MT))
       return MT;
     if (D.getVFS().exists(SysRoot + "/lib/mips64-linux-gnu"))
       return "mips64-linux-gnu";
     break;
   }
   case llvm::Triple::mips64el: {
     if (IsAndroid)
       return "mips64el-linux-android";
     std::string MT = std::string(IsMipsR6 ? "mipsisa64r6el" : "mips64el") +
                      "-linux-" + (IsMipsN32Abi ? "gnuabin32" : "gnuabi64");
     if (D.getVFS().exists(SysRoot + "/lib/" + MT))
       return MT;
     if (D.getVFS().exists(SysRoot + "/lib/mips64el-linux-gnu"))
       return "mips64el-linux-gnu";
     break;
   }
   case llvm::Triple::ppc:
     if (D.getVFS().exists(SysRoot + "/lib/powerpc-linux-gnuspe"))
       return "powerpc-linux-gnuspe";
     return "powerpc-linux-gnu";
   case llvm::Triple::ppcle:
     return "powerpcle-linux-gnu";
   case llvm::Triple::ppc64:
     return "powerpc64-linux-gnu";
   case llvm::Triple::ppc64le:
     return "powerpc64le-linux-gnu";
   case llvm::Triple::sparc:
     return "sparc-linux-gnu";
   case llvm::Triple::sparcv9:
     return "sparc64-linux-gnu";
   case llvm::Triple::systemz:
     return "s390x-linux-gnu";
   }
   return TargetTriple.str();
 }
 
 static StringRef getOSLibDir(const llvm::Triple &Triple, const ArgList &Args) {
   if (Triple.isMIPS()) {
     if (Triple.isAndroid()) {
       StringRef CPUName;
       StringRef ABIName;
       tools::mips::getMipsCPUAndABI(Args, Triple, CPUName, ABIName);
       if (CPUName == "mips32r6")
         return "libr6";
       if (CPUName == "mips32r2")
         return "libr2";
     }
     // lib32 directory has a special meaning on MIPS targets.
     // It contains N32 ABI binaries. Use this folder if produce
     // code for N32 ABI only.
     if (tools::mips::hasMipsAbiArg(Args, "n32"))
       return "lib32";
     return Triple.isArch32Bit() ? "lib" : "lib64";
   }
 
   // It happens that only x86, PPC and SPARC use the 'lib32' variant of
   // oslibdir, and using that variant while targeting other architectures causes
   // problems because the libraries are laid out in shared system roots that
   // can't cope with a 'lib32' library search path being considered. So we only
   // enable them when we know we may need it.
   //
   // FIXME: This is a bit of a hack. We should really unify this code for
   // reasoning about oslibdir spellings with the lib dir spellings in the
   // GCCInstallationDetector, but that is a more significant refactoring.
   if (Triple.getArch() == llvm::Triple::x86 || Triple.isPPC32() ||
       Triple.getArch() == llvm::Triple::sparc)
     return "lib32";
 
   if (Triple.getArch() == llvm::Triple::x86_64 && Triple.isX32())
     return "libx32";
 
   if (Triple.getArch() == llvm::Triple::riscv32)
     return "lib32";
 
   return Triple.isArch32Bit() ? "lib" : "lib64";
 }
 
 Linux::Linux(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
     : Generic_ELF(D, Triple, Args) {
   GCCInstallation.init(Triple, Args);
   Multilibs = GCCInstallation.getMultilibs();
   SelectedMultilib = GCCInstallation.getMultilib();
   llvm::Triple::ArchType Arch = Triple.getArch();
   std::string SysRoot = computeSysRoot();
   ToolChain::path_list &PPaths = getProgramPaths();
 
   Generic_GCC::PushPPaths(PPaths);
 
   Distro Distro(D.getVFS(), Triple);
 
   if (Distro.IsAlpineLinux() || Triple.isAndroid()) {
     ExtraOpts.push_back("-z");
     ExtraOpts.push_back("now");
   }
 
   if (Distro.IsOpenSUSE() || Distro.IsUbuntu() || Distro.IsAlpineLinux() ||
       Triple.isAndroid()) {
     ExtraOpts.push_back("-z");
     ExtraOpts.push_back("relro");
   }
 
   // Android ARM/AArch64 use max-page-size=4096 to reduce VMA usage. Note, lld
   // from 11 onwards default max-page-size to 65536 for both ARM and AArch64.
   if ((Triple.isARM() || Triple.isAArch64()) && Triple.isAndroid()) {
     ExtraOpts.push_back("-z");
     ExtraOpts.push_back("max-page-size=4096");
   }
 
   if (GCCInstallation.getParentLibPath().contains("opt/rh/"))
     // With devtoolset on RHEL, we want to add a bin directory that is relative
     // to the detected gcc install, because if we are using devtoolset gcc then
     // we want to use other tools from devtoolset (e.g. ld) instead of the
     // standard system tools.
     PPaths.push_back(Twine(GCCInstallation.getParentLibPath() +
                      "/../bin").str());
 
   if (Arch == llvm::Triple::arm || Arch == llvm::Triple::thumb)
     ExtraOpts.push_back("-X");
 
   const bool IsAndroid = Triple.isAndroid();
   const bool IsMips = Triple.isMIPS();
   const bool IsHexagon = Arch == llvm::Triple::hexagon;
   const bool IsRISCV = Triple.isRISCV();
 
   if (IsMips && !SysRoot.empty())
     ExtraOpts.push_back("--sysroot=" + SysRoot);
 
   // Do not use 'gnu' hash style for Mips targets because .gnu.hash
   // and the MIPS ABI require .dynsym to be sorted in different ways.
   // .gnu.hash needs symbols to be grouped by hash code whereas the MIPS
   // ABI requires a mapping between the GOT and the symbol table.
   // Android loader does not support .gnu.hash until API 23.
   // Hexagon linker/loader does not support .gnu.hash
   if (!IsMips && !IsHexagon) {
     if (Distro.IsRedhat() || Distro.IsOpenSUSE() || Distro.IsAlpineLinux() ||
         (Distro.IsUbuntu() && Distro >= Distro::UbuntuMaverick) ||
         (IsAndroid && !Triple.isAndroidVersionLT(23)))
       ExtraOpts.push_back("--hash-style=gnu");
 
     if (Distro.IsDebian() || Distro.IsOpenSUSE() ||
         Distro == Distro::UbuntuLucid || Distro == Distro::UbuntuJaunty ||
         Distro == Distro::UbuntuKarmic ||
         (IsAndroid && Triple.isAndroidVersionLT(23)))
       ExtraOpts.push_back("--hash-style=both");
   }
 
 #ifdef ENABLE_LINKER_BUILD_ID
   ExtraOpts.push_back("--build-id");
 #endif
 
   if (IsAndroid || Distro.IsOpenSUSE())
     ExtraOpts.push_back("--enable-new-dtags");
 
   // The selection of paths to try here is designed to match the patterns which
   // the GCC driver itself uses, as this is part of the GCC-compatible driver.
   // This was determined by running GCC in a fake filesystem, creating all
   // possible permutations of these directories, and seeing which ones it added
   // to the link paths.
   path_list &Paths = getFilePaths();
 
   const std::string OSLibDir = std::string(getOSLibDir(Triple, Args));
   const std::string MultiarchTriple = getMultiarchTriple(D, Triple, SysRoot);
 
   // mips32: Debian multilib, we use /libo32, while in other case, /lib is
   // used. We need add both libo32 and /lib.
   if (Arch == llvm::Triple::mips || Arch == llvm::Triple::mipsel) {
     Generic_GCC::AddMultilibPaths(D, SysRoot, "libo32", MultiarchTriple, Paths);
     addPathIfExists(D, SysRoot + "/libo32", Paths);
     addPathIfExists(D, SysRoot + "/usr/libo32", Paths);
   }
   Generic_GCC::AddMultilibPaths(D, SysRoot, OSLibDir, MultiarchTriple, Paths);
 
   addPathIfExists(D, SysRoot + "/lib/" + MultiarchTriple, Paths);
   addPathIfExists(D, SysRoot + "/lib/../" + OSLibDir, Paths);
 
   if (IsAndroid) {
     // Android sysroots contain a library directory for each supported OS
     // version as well as some unversioned libraries in the usual multiarch
     // directory.
     addPathIfExists(
         D,
         SysRoot + "/usr/lib/" + MultiarchTriple + "/" +
             llvm::to_string(Triple.getEnvironmentVersion().getMajor()),
         Paths);
   }
 
   addPathIfExists(D, SysRoot + "/usr/lib/" + MultiarchTriple, Paths);
   // 64-bit OpenEmbedded sysroots may not have a /usr/lib dir. So they cannot
   // find /usr/lib64 as it is referenced as /usr/lib/../lib64. So we handle
   // this here.
   if (Triple.getVendor() == llvm::Triple::OpenEmbedded &&
       Triple.isArch64Bit())
     addPathIfExists(D, SysRoot + "/usr/" + OSLibDir, Paths);
   else
     addPathIfExists(D, SysRoot + "/usr/lib/../" + OSLibDir, Paths);
   if (IsRISCV) {
     StringRef ABIName = tools::riscv::getRISCVABI(Args, Triple);
     addPathIfExists(D, SysRoot + "/" + OSLibDir + "/" + ABIName, Paths);
     addPathIfExists(D, SysRoot + "/usr/" + OSLibDir + "/" + ABIName, Paths);
   }
 
   Generic_GCC::AddMultiarchPaths(D, SysRoot, OSLibDir, Paths);
 
-  // Similar to the logic for GCC above, if we are currently running Clang
-  // inside of the requested system root, add its parent library path to those
-  // searched.
-  // FIXME: It's not clear whether we should use the driver's installed
-  // directory ('Dir' below) or the ResourceDir.
-  if (StringRef(D.Dir).startswith(SysRoot)) {
-    // Even if OSLibDir != "lib", this is needed for Clang in the build
-    // directory (not installed) to find libc++.
+  // The deprecated -DLLVM_ENABLE_PROJECTS=libcxx configuration installs
+  // libc++.so in D.Dir+"/../lib/". Detect this path.
+  // TODO Remove once LLVM_ENABLE_PROJECTS=libcxx is unsupported.
+  if (StringRef(D.Dir).startswith(SysRoot) &&
+      D.getVFS().exists(D.Dir + "/../lib/libc++.so"))
     addPathIfExists(D, D.Dir + "/../lib", Paths);
-    if (OSLibDir != "lib")
-      addPathIfExists(D, D.Dir + "/../" + OSLibDir, Paths);
-  }
 
   addPathIfExists(D, SysRoot + "/lib", Paths);
   addPathIfExists(D, SysRoot + "/usr/lib", Paths);
 }
 
 ToolChain::RuntimeLibType Linux::GetDefaultRuntimeLibType() const {
   if (getTriple().isAndroid())
     return ToolChain::RLT_CompilerRT;
   return Generic_ELF::GetDefaultRuntimeLibType();
 }
 
 unsigned Linux::GetDefaultDwarfVersion() const {
   if (getTriple().isAndroid())
     return 4;
   return ToolChain::GetDefaultDwarfVersion();
 }
 
 ToolChain::CXXStdlibType Linux::GetDefaultCXXStdlibType() const {
   if (getTriple().isAndroid())
     return ToolChain::CST_Libcxx;
   return ToolChain::CST_Libstdcxx;
 }
 
 bool Linux::HasNativeLLVMSupport() const { return true; }
 
 Tool *Linux::buildLinker() const { return new tools::gnutools::Linker(*this); }
 
 Tool *Linux::buildStaticLibTool() const {
   return new tools::gnutools::StaticLibTool(*this);
 }
 
 Tool *Linux::buildAssembler() const {
   return new tools::gnutools::Assembler(*this);
 }
 
 std::string Linux::computeSysRoot() const {
   if (!getDriver().SysRoot.empty())
     return getDriver().SysRoot;
 
   if (getTriple().isAndroid()) {
     // Android toolchains typically include a sysroot at ../sysroot relative to
     // the clang binary.
     const StringRef ClangDir = getDriver().getInstalledDir();
     std::string AndroidSysRootPath = (ClangDir + "/../sysroot").str();
     if (getVFS().exists(AndroidSysRootPath))
       return AndroidSysRootPath;
   }
 
   if (!GCCInstallation.isValid() || !getTriple().isMIPS())
     return std::string();
 
   // Standalone MIPS toolchains use different names for sysroot folder
   // and put it into different places. Here we try to check some known
   // variants.
 
   const StringRef InstallDir = GCCInstallation.getInstallPath();
   const StringRef TripleStr = GCCInstallation.getTriple().str();
   const Multilib &Multilib = GCCInstallation.getMultilib();
 
   std::string Path =
       (InstallDir + "/../../../../" + TripleStr + "/libc" + Multilib.osSuffix())
           .str();
 
   if (getVFS().exists(Path))
     return Path;
 
   Path = (InstallDir + "/../../../../sysroot" + Multilib.osSuffix()).str();
 
   if (getVFS().exists(Path))
     return Path;
 
   return std::string();
 }
 
 std::string Linux::getDynamicLinker(const ArgList &Args) const {
   const llvm::Triple::ArchType Arch = getArch();
   const llvm::Triple &Triple = getTriple();
 
   const Distro Distro(getDriver().getVFS(), Triple);
 
   if (Triple.isAndroid())
     return Triple.isArch64Bit() ? "/system/bin/linker64" : "/system/bin/linker";
 
   if (Triple.isMusl()) {
     std::string ArchName;
     bool IsArm = false;
 
     switch (Arch) {
     case llvm::Triple::arm:
     case llvm::Triple::thumb:
       ArchName = "arm";
       IsArm = true;
       break;
     case llvm::Triple::armeb:
     case llvm::Triple::thumbeb:
       ArchName = "armeb";
       IsArm = true;
       break;
     case llvm::Triple::x86:
       ArchName = "i386";
       break;
     case llvm::Triple::x86_64:
       ArchName = Triple.isX32() ? "x32" : Triple.getArchName().str();
       break;
     default:
       ArchName = Triple.getArchName().str();
     }
     if (IsArm &&
         (Triple.getEnvironment() == llvm::Triple::MuslEABIHF ||
          tools::arm::getARMFloatABI(*this, Args) == tools::arm::FloatABI::Hard))
       ArchName += "hf";
     if (Arch == llvm::Triple::ppc &&
         Triple.getSubArch() == llvm::Triple::PPCSubArch_spe)
       ArchName = "powerpc-sf";
 
     return "/lib/ld-musl-" + ArchName + ".so.1";
   }
 
   std::string LibDir;
   std::string Loader;
 
   switch (Arch) {
   default:
     llvm_unreachable("unsupported architecture");
 
   case llvm::Triple::aarch64:
     LibDir = "lib";
     Loader = "ld-linux-aarch64.so.1";
     break;
   case llvm::Triple::aarch64_be:
     LibDir = "lib";
     Loader = "ld-linux-aarch64_be.so.1";
     break;
   case llvm::Triple::arm:
   case llvm::Triple::thumb:
   case llvm::Triple::armeb:
   case llvm::Triple::thumbeb: {
     const bool HF =
         Triple.getEnvironment() == llvm::Triple::GNUEABIHF ||
         tools::arm::getARMFloatABI(*this, Args) == tools::arm::FloatABI::Hard;
 
     LibDir = "lib";
     Loader = HF ? "ld-linux-armhf.so.3" : "ld-linux.so.3";
     break;
   }
   case llvm::Triple::m68k:
     LibDir = "lib";
     Loader = "ld.so.1";
     break;
   case llvm::Triple::mips:
   case llvm::Triple::mipsel:
   case llvm::Triple::mips64:
   case llvm::Triple::mips64el: {
     bool IsNaN2008 = tools::mips::isNaN2008(getDriver(), Args, Triple);
 
     LibDir = "lib" + tools::mips::getMipsABILibSuffix(Args, Triple);
 
     if (tools::mips::isUCLibc(Args))
       Loader = IsNaN2008 ? "ld-uClibc-mipsn8.so.0" : "ld-uClibc.so.0";
     else if (!Triple.hasEnvironment() &&
              Triple.getVendor() == llvm::Triple::VendorType::MipsTechnologies)
       Loader =
           Triple.isLittleEndian() ? "ld-musl-mipsel.so.1" : "ld-musl-mips.so.1";
     else
       Loader = IsNaN2008 ? "ld-linux-mipsn8.so.1" : "ld.so.1";
 
     break;
   }
   case llvm::Triple::ppc:
     LibDir = "lib";
     Loader = "ld.so.1";
     break;
   case llvm::Triple::ppcle:
     LibDir = "lib";
     Loader = "ld.so.1";
     break;
   case llvm::Triple::ppc64:
     LibDir = "lib64";
     Loader =
         (tools::ppc::hasPPCAbiArg(Args, "elfv2")) ? "ld64.so.2" : "ld64.so.1";
     break;
   case llvm::Triple::ppc64le:
     LibDir = "lib64";
     Loader =
         (tools::ppc::hasPPCAbiArg(Args, "elfv1")) ? "ld64.so.1" : "ld64.so.2";
     break;
   case llvm::Triple::riscv32: {
     StringRef ABIName = tools::riscv::getRISCVABI(Args, Triple);
     LibDir = "lib";
     Loader = ("ld-linux-riscv32-" + ABIName + ".so.1").str();
     break;
   }
   case llvm::Triple::riscv64: {
     StringRef ABIName = tools::riscv::getRISCVABI(Args, Triple);
     LibDir = "lib";
     Loader = ("ld-linux-riscv64-" + ABIName + ".so.1").str();
     break;
   }
   case llvm::Triple::sparc:
   case llvm::Triple::sparcel:
     LibDir = "lib";
     Loader = "ld-linux.so.2";
     break;
   case llvm::Triple::sparcv9:
     LibDir = "lib64";
     Loader = "ld-linux.so.2";
     break;
   case llvm::Triple::systemz:
     LibDir = "lib";
     Loader = "ld64.so.1";
     break;
   case llvm::Triple::x86:
     LibDir = "lib";
     Loader = "ld-linux.so.2";
     break;
   case llvm::Triple::x86_64: {
     bool X32 = Triple.isX32();
 
     LibDir = X32 ? "libx32" : "lib64";
     Loader = X32 ? "ld-linux-x32.so.2" : "ld-linux-x86-64.so.2";
     break;
   }
   case llvm::Triple::ve:
     return "/opt/nec/ve/lib/ld-linux-ve.so.1";
   }
 
   if (Distro == Distro::Exherbo &&
       (Triple.getVendor() == llvm::Triple::UnknownVendor ||
        Triple.getVendor() == llvm::Triple::PC))
     return "/usr/" + Triple.str() + "/lib/" + Loader;
   return "/" + LibDir + "/" + Loader;
 }
 
 void Linux::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
                                       ArgStringList &CC1Args) const {
   const Driver &D = getDriver();
   std::string SysRoot = computeSysRoot();
 
   if (DriverArgs.hasArg(clang::driver::options::OPT_nostdinc))
     return;
 
   // Add 'include' in the resource directory, which is similar to
   // GCC_INCLUDE_DIR (private headers) in GCC. Note: the include directory
   // contains some files conflicting with system /usr/include. musl systems
   // prefer the /usr/include copies which are more relevant.
   SmallString<128> ResourceDirInclude(D.ResourceDir);
   llvm::sys::path::append(ResourceDirInclude, "include");
   if (!DriverArgs.hasArg(options::OPT_nobuiltininc) &&
       (!getTriple().isMusl() || DriverArgs.hasArg(options::OPT_nostdlibinc)))
     addSystemInclude(DriverArgs, CC1Args, ResourceDirInclude);
 
   if (DriverArgs.hasArg(options::OPT_nostdlibinc))
     return;
 
   // LOCAL_INCLUDE_DIR
   addSystemInclude(DriverArgs, CC1Args, SysRoot + "/usr/local/include");
   // TOOL_INCLUDE_DIR
   AddMultilibIncludeArgs(DriverArgs, CC1Args);
 
   // Check for configure-time C include directories.
   StringRef CIncludeDirs(C_INCLUDE_DIRS);
   if (CIncludeDirs != "") {
     SmallVector<StringRef, 5> dirs;
     CIncludeDirs.split(dirs, ":");
     for (StringRef dir : dirs) {
       StringRef Prefix =
           llvm::sys::path::is_absolute(dir) ? "" : StringRef(SysRoot);
       addExternCSystemInclude(DriverArgs, CC1Args, Prefix + dir);
     }
     return;
   }
 
   // On systems using multiarch and Android, add /usr/include/$triple before
   // /usr/include.
   std::string MultiarchIncludeDir = getMultiarchTriple(D, getTriple(), SysRoot);
   if (!MultiarchIncludeDir.empty() &&
       D.getVFS().exists(SysRoot + "/usr/include/" + MultiarchIncludeDir))
     addExternCSystemInclude(DriverArgs, CC1Args,
                             SysRoot + "/usr/include/" + MultiarchIncludeDir);
 
   if (getTriple().getOS() == llvm::Triple::RTEMS)
     return;
 
   // Add an include of '/include' directly. This isn't provided by default by
   // system GCCs, but is often used with cross-compiling GCCs, and harmless to
   // add even when Clang is acting as-if it were a system compiler.
   addExternCSystemInclude(DriverArgs, CC1Args, SysRoot + "/include");
 
   addExternCSystemInclude(DriverArgs, CC1Args, SysRoot + "/usr/include");
 
   if (!DriverArgs.hasArg(options::OPT_nobuiltininc) && getTriple().isMusl())
     addSystemInclude(DriverArgs, CC1Args, ResourceDirInclude);
 }
 
 void Linux::addLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
                                      llvm::opt::ArgStringList &CC1Args) const {
   // We need a detected GCC installation on Linux to provide libstdc++'s
   // headers in odd Linuxish places.
   if (!GCCInstallation.isValid())
     return;
 
   // Detect Debian g++-multiarch-incdir.diff.
   StringRef TripleStr = GCCInstallation.getTriple().str();
   StringRef DebianMultiarch =
       GCCInstallation.getTriple().getArch() == llvm::Triple::x86
           ? "i386-linux-gnu"
           : TripleStr;
 
   // Try generic GCC detection first.
   if (Generic_GCC::addGCCLibStdCxxIncludePaths(DriverArgs, CC1Args,
                                                DebianMultiarch))
     return;
 
   StringRef LibDir = GCCInstallation.getParentLibPath();
   const Multilib &Multilib = GCCInstallation.getMultilib();
   const GCCVersion &Version = GCCInstallation.getVersion();
 
   const std::string LibStdCXXIncludePathCandidates[] = {
       // Android standalone toolchain has C++ headers in yet another place.
       LibDir.str() + "/../" + TripleStr.str() + "/include/c++/" + Version.Text,
       // Freescale SDK C++ headers are directly in <sysroot>/usr/include/c++,
       // without a subdirectory corresponding to the gcc version.
       LibDir.str() + "/../include/c++",
       // Cray's gcc installation puts headers under "g++" without a
       // version suffix.
       LibDir.str() + "/../include/g++",
   };
 
   for (const auto &IncludePath : LibStdCXXIncludePathCandidates) {
     if (addLibStdCXXIncludePaths(IncludePath, TripleStr,
                                  Multilib.includeSuffix(), DriverArgs, CC1Args))
       break;
   }
 }
 
 void Linux::AddCudaIncludeArgs(const ArgList &DriverArgs,
                                ArgStringList &CC1Args) const {
   CudaInstallation.AddCudaIncludeArgs(DriverArgs, CC1Args);
 }
 
 void Linux::AddHIPIncludeArgs(const ArgList &DriverArgs,
                               ArgStringList &CC1Args) const {
   RocmInstallation.AddHIPIncludeArgs(DriverArgs, CC1Args);
 }
 
 void Linux::AddIAMCUIncludeArgs(const ArgList &DriverArgs,
                                 ArgStringList &CC1Args) const {
   if (GCCInstallation.isValid()) {
     CC1Args.push_back("-isystem");
     CC1Args.push_back(DriverArgs.MakeArgString(
         GCCInstallation.getParentLibPath() + "/../" +
         GCCInstallation.getTriple().str() + "/include"));
   }
 }
 
 bool Linux::isPIEDefault(const llvm::opt::ArgList &Args) const {
   return CLANG_DEFAULT_PIE_ON_LINUX || getTriple().isAndroid() ||
          getTriple().isMusl() || getSanitizerArgs(Args).requiresPIE();
 }
 
 bool Linux::IsAArch64OutlineAtomicsDefault(const ArgList &Args) const {
   // Outline atomics for AArch64 are supported by compiler-rt
   // and libgcc since 9.3.1
   assert(getTriple().isAArch64() && "expected AArch64 target!");
   ToolChain::RuntimeLibType RtLib = GetRuntimeLibType(Args);
   if (RtLib == ToolChain::RLT_CompilerRT)
     return true;
   assert(RtLib == ToolChain::RLT_Libgcc && "unexpected runtime library type!");
   if (GCCInstallation.getVersion().isOlderThan(9, 3, 1))
     return false;
   return true;
 }
 
 bool Linux::IsMathErrnoDefault() const {
   if (getTriple().isAndroid() || getTriple().isMusl())
     return false;
   return Generic_ELF::IsMathErrnoDefault();
 }
 
 SanitizerMask Linux::getSupportedSanitizers() const {
   const bool IsX86 = getTriple().getArch() == llvm::Triple::x86;
   const bool IsX86_64 = getTriple().getArch() == llvm::Triple::x86_64;
   const bool IsMIPS = getTriple().isMIPS32();
   const bool IsMIPS64 = getTriple().isMIPS64();
   const bool IsPowerPC64 = getTriple().getArch() == llvm::Triple::ppc64 ||
                            getTriple().getArch() == llvm::Triple::ppc64le;
   const bool IsAArch64 = getTriple().getArch() == llvm::Triple::aarch64 ||
                          getTriple().getArch() == llvm::Triple::aarch64_be;
   const bool IsArmArch = getTriple().getArch() == llvm::Triple::arm ||
                          getTriple().getArch() == llvm::Triple::thumb ||
                          getTriple().getArch() == llvm::Triple::armeb ||
                          getTriple().getArch() == llvm::Triple::thumbeb;
   const bool IsRISCV64 = getTriple().getArch() == llvm::Triple::riscv64;
   const bool IsSystemZ = getTriple().getArch() == llvm::Triple::systemz;
   const bool IsHexagon = getTriple().getArch() == llvm::Triple::hexagon;
   SanitizerMask Res = ToolChain::getSupportedSanitizers();
   Res |= SanitizerKind::Address;
   Res |= SanitizerKind::PointerCompare;
   Res |= SanitizerKind::PointerSubtract;
   Res |= SanitizerKind::Fuzzer;
   Res |= SanitizerKind::FuzzerNoLink;
   Res |= SanitizerKind::KernelAddress;
   Res |= SanitizerKind::Memory;
   Res |= SanitizerKind::Vptr;
   Res |= SanitizerKind::SafeStack;
   if (IsX86_64 || IsMIPS64 || IsAArch64)
     Res |= SanitizerKind::DataFlow;
   if (IsX86_64 || IsMIPS64 || IsAArch64 || IsX86 || IsArmArch || IsPowerPC64 ||
       IsRISCV64 || IsSystemZ || IsHexagon)
     Res |= SanitizerKind::Leak;
   if (IsX86_64 || IsMIPS64 || IsAArch64 || IsPowerPC64 || IsSystemZ)
     Res |= SanitizerKind::Thread;
   if (IsX86_64)
     Res |= SanitizerKind::KernelMemory;
   if (IsX86 || IsX86_64)
     Res |= SanitizerKind::Function;
   if (IsX86_64 || IsMIPS64 || IsAArch64 || IsX86 || IsMIPS || IsArmArch ||
       IsPowerPC64 || IsHexagon)
     Res |= SanitizerKind::Scudo;
   if (IsX86_64 || IsAArch64) {
     Res |= SanitizerKind::HWAddress;
     Res |= SanitizerKind::KernelHWAddress;
   }
   return Res;
 }
 
 void Linux::addProfileRTLibs(const llvm::opt::ArgList &Args,
                              llvm::opt::ArgStringList &CmdArgs) const {
   // Add linker option -u__llvm_profile_runtime to cause runtime
   // initialization module to be linked in.
   if (needsProfileRT(Args))
     CmdArgs.push_back(Args.MakeArgString(
         Twine("-u", llvm::getInstrProfRuntimeHookVarName())));
   ToolChain::addProfileRTLibs(Args, CmdArgs);
 }
 
 llvm::DenormalMode
 Linux::getDefaultDenormalModeForType(const llvm::opt::ArgList &DriverArgs,
                                      const JobAction &JA,
                                      const llvm::fltSemantics *FPType) const {
   switch (getTriple().getArch()) {
   case llvm::Triple::x86:
   case llvm::Triple::x86_64: {
     std::string Unused;
     // DAZ and FTZ are turned on in crtfastmath.o
     if (!DriverArgs.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles) &&
         isFastMathRuntimeAvailable(DriverArgs, Unused))
       return llvm::DenormalMode::getPreserveSign();
     return llvm::DenormalMode::getIEEE();
   }
   default:
     return llvm::DenormalMode::getIEEE();
   }
 }
 
 void Linux::addExtraOpts(llvm::opt::ArgStringList &CmdArgs) const {
   for (const auto &Opt : ExtraOpts)
     CmdArgs.push_back(Opt.c_str());
 }
diff --git a/contrib/llvm-project/clang/lib/Driver/ToolChains/Solaris.cpp b/contrib/llvm-project/clang/lib/Driver/ToolChains/Solaris.cpp
index 24f18b92dd66..cb3898575d3a 100644
--- a/contrib/llvm-project/clang/lib/Driver/ToolChains/Solaris.cpp
+++ b/contrib/llvm-project/clang/lib/Driver/ToolChains/Solaris.cpp
@@ -1,296 +1,300 @@
 //===--- Solaris.cpp - Solaris ToolChain Implementations --------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "Solaris.h"
 #include "CommonArgs.h"
 #include "clang/Basic/LangStandard.h"
 #include "clang/Config/config.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/DriverDiagnostic.h"
 #include "clang/Driver/Options.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 
 using namespace clang::driver;
 using namespace clang::driver::tools;
 using namespace clang::driver::toolchains;
 using namespace clang;
 using namespace llvm::opt;
 
 void solaris::Assembler::ConstructJob(Compilation &C, const JobAction &JA,
                                       const InputInfo &Output,
                                       const InputInfoList &Inputs,
                                       const ArgList &Args,
                                       const char *LinkingOutput) const {
   claimNoWarnArgs(Args);
   ArgStringList CmdArgs;
 
   Args.AddAllArgValues(CmdArgs, options::OPT_Wa_COMMA, options::OPT_Xassembler);
 
   CmdArgs.push_back("-o");
   CmdArgs.push_back(Output.getFilename());
 
   for (const auto &II : Inputs)
     CmdArgs.push_back(II.getFilename());
 
   const char *Exec = Args.MakeArgString(getToolChain().GetProgramPath("as"));
   C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
                                          Exec, CmdArgs, Inputs, Output));
 }
 
 void solaris::Linker::ConstructJob(Compilation &C, const JobAction &JA,
                                    const InputInfo &Output,
                                    const InputInfoList &Inputs,
                                    const ArgList &Args,
                                    const char *LinkingOutput) const {
   ArgStringList CmdArgs;
 
   // Demangle C++ names in errors
   CmdArgs.push_back("-C");
 
   if (!Args.hasArg(options::OPT_nostdlib, options::OPT_shared)) {
     CmdArgs.push_back("-e");
     CmdArgs.push_back("_start");
   }
 
   if (Args.hasArg(options::OPT_static)) {
     CmdArgs.push_back("-Bstatic");
     CmdArgs.push_back("-dn");
   } else {
     CmdArgs.push_back("-Bdynamic");
     if (Args.hasArg(options::OPT_shared)) {
       CmdArgs.push_back("-shared");
     }
 
     // libpthread has been folded into libc since Solaris 10, no need to do
     // anything for pthreads. Claim argument to avoid warning.
     Args.ClaimAllArgs(options::OPT_pthread);
     Args.ClaimAllArgs(options::OPT_pthreads);
   }
 
   if (Output.isFilename()) {
     CmdArgs.push_back("-o");
     CmdArgs.push_back(Output.getFilename());
   } else {
     assert(Output.isNothing() && "Invalid output.");
   }
 
-  if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles)) {
+  if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles,
+                   options::OPT_r)) {
     if (!Args.hasArg(options::OPT_shared))
       CmdArgs.push_back(
           Args.MakeArgString(getToolChain().GetFilePath("crt1.o")));
 
     CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath("crti.o")));
 
     const Arg *Std = Args.getLastArg(options::OPT_std_EQ, options::OPT_ansi);
     bool HaveAnsi = false;
     const LangStandard *LangStd = nullptr;
     if (Std) {
       HaveAnsi = Std->getOption().matches(options::OPT_ansi);
       if (!HaveAnsi)
         LangStd = LangStandard::getLangStandardForName(Std->getValue());
     }
 
     const char *values_X = "values-Xa.o";
     // Use values-Xc.o for -ansi, -std=c*, -std=iso9899:199409.
     if (HaveAnsi || (LangStd && !LangStd->isGNUMode()))
       values_X = "values-Xc.o";
     CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(values_X)));
 
     const char *values_xpg = "values-xpg6.o";
     // Use values-xpg4.o for -std=c90, -std=gnu90, -std=iso9899:199409.
     if (LangStd && LangStd->getLanguage() == Language::C && !LangStd->isC99())
       values_xpg = "values-xpg4.o";
     CmdArgs.push_back(
         Args.MakeArgString(getToolChain().GetFilePath(values_xpg)));
     CmdArgs.push_back(
         Args.MakeArgString(getToolChain().GetFilePath("crtbegin.o")));
   }
 
   getToolChain().AddFilePathLibArgs(Args, CmdArgs);
 
   Args.AddAllArgs(CmdArgs, {options::OPT_L, options::OPT_T_Group,
                             options::OPT_e, options::OPT_r});
 
   bool NeedsSanitizerDeps = addSanitizerRuntimes(getToolChain(), Args, CmdArgs);
   AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs, JA);
 
-  if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {
+  if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs,
+                   options::OPT_r)) {
     if (getToolChain().ShouldLinkCXXStdlib(Args))
       getToolChain().AddCXXStdlibLibArgs(Args, CmdArgs);
     if (Args.hasArg(options::OPT_fstack_protector) ||
         Args.hasArg(options::OPT_fstack_protector_strong) ||
         Args.hasArg(options::OPT_fstack_protector_all)) {
       // Explicitly link ssp libraries, not folded into Solaris libc.
       CmdArgs.push_back("-lssp_nonshared");
       CmdArgs.push_back("-lssp");
     }
     // LLVM support for atomics on 32-bit SPARC V8+ is incomplete, so
     // forcibly link with libatomic as a workaround.
     if (getToolChain().getTriple().getArch() == llvm::Triple::sparc) {
       CmdArgs.push_back(getAsNeededOption(getToolChain(), true));
       CmdArgs.push_back("-latomic");
       CmdArgs.push_back(getAsNeededOption(getToolChain(), false));
     }
     CmdArgs.push_back("-lgcc_s");
     CmdArgs.push_back("-lc");
     if (!Args.hasArg(options::OPT_shared)) {
       CmdArgs.push_back("-lgcc");
       CmdArgs.push_back("-lm");
     }
     if (NeedsSanitizerDeps)
       linkSanitizerRuntimeDeps(getToolChain(), CmdArgs);
   }
 
-  if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles)) {
+  if (!Args.hasArg(options::OPT_nostdlib, options::OPT_nostartfiles,
+                   options::OPT_r)) {
     CmdArgs.push_back(
         Args.MakeArgString(getToolChain().GetFilePath("crtend.o")));
+    CmdArgs.push_back(
+        Args.MakeArgString(getToolChain().GetFilePath("crtn.o")));
   }
-  CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath("crtn.o")));
 
   getToolChain().addProfileRTLibs(Args, CmdArgs);
 
   const char *Exec = Args.MakeArgString(getToolChain().GetLinkerPath());
   C.addCommand(std::make_unique<Command>(JA, *this, ResponseFileSupport::None(),
                                          Exec, CmdArgs, Inputs, Output));
 }
 
 static StringRef getSolarisLibSuffix(const llvm::Triple &Triple) {
   switch (Triple.getArch()) {
   case llvm::Triple::x86:
   case llvm::Triple::sparc:
     break;
   case llvm::Triple::x86_64:
     return "/amd64";
   case llvm::Triple::sparcv9:
     return "/sparcv9";
   default:
     llvm_unreachable("Unsupported architecture");
   }
   return "";
 }
 
 /// Solaris - Solaris tool chain which can call as(1) and ld(1) directly.
 
 Solaris::Solaris(const Driver &D, const llvm::Triple &Triple,
                  const ArgList &Args)
     : Generic_ELF(D, Triple, Args) {
 
   GCCInstallation.init(Triple, Args);
 
   StringRef LibSuffix = getSolarisLibSuffix(Triple);
   path_list &Paths = getFilePaths();
   if (GCCInstallation.isValid()) {
     // On Solaris gcc uses both an architecture-specific path with triple in it
     // as well as a more generic lib path (+arch suffix).
     addPathIfExists(D,
                     GCCInstallation.getInstallPath() +
                         GCCInstallation.getMultilib().gccSuffix(),
                     Paths);
     addPathIfExists(D, GCCInstallation.getParentLibPath() + LibSuffix, Paths);
   }
 
   // If we are currently running Clang inside of the requested system root,
   // add its parent library path to those searched.
   if (StringRef(D.Dir).startswith(D.SysRoot))
     addPathIfExists(D, D.Dir + "/../lib", Paths);
 
   addPathIfExists(D, D.SysRoot + "/usr/lib" + LibSuffix, Paths);
 }
 
 SanitizerMask Solaris::getSupportedSanitizers() const {
   const bool IsX86 = getTriple().getArch() == llvm::Triple::x86;
   const bool IsX86_64 = getTriple().getArch() == llvm::Triple::x86_64;
   SanitizerMask Res = ToolChain::getSupportedSanitizers();
   // FIXME: Omit X86_64 until 64-bit support is figured out.
   if (IsX86) {
     Res |= SanitizerKind::Address;
     Res |= SanitizerKind::PointerCompare;
     Res |= SanitizerKind::PointerSubtract;
   }
   if (IsX86 || IsX86_64)
     Res |= SanitizerKind::Function;
   Res |= SanitizerKind::Vptr;
   return Res;
 }
 
 Tool *Solaris::buildAssembler() const {
   return new tools::solaris::Assembler(*this);
 }
 
 Tool *Solaris::buildLinker() const { return new tools::solaris::Linker(*this); }
 
 void Solaris::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
                                         ArgStringList &CC1Args) const {
   const Driver &D = getDriver();
 
   if (DriverArgs.hasArg(clang::driver::options::OPT_nostdinc))
     return;
 
   if (!DriverArgs.hasArg(options::OPT_nostdlibinc))
     addSystemInclude(DriverArgs, CC1Args, D.SysRoot + "/usr/local/include");
 
   if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {
     SmallString<128> P(D.ResourceDir);
     llvm::sys::path::append(P, "include");
     addSystemInclude(DriverArgs, CC1Args, P);
   }
 
   if (DriverArgs.hasArg(options::OPT_nostdlibinc))
     return;
 
   // Check for configure-time C include directories.
   StringRef CIncludeDirs(C_INCLUDE_DIRS);
   if (CIncludeDirs != "") {
     SmallVector<StringRef, 5> dirs;
     CIncludeDirs.split(dirs, ":");
     for (StringRef dir : dirs) {
       StringRef Prefix =
           llvm::sys::path::is_absolute(dir) ? "" : StringRef(D.SysRoot);
       addExternCSystemInclude(DriverArgs, CC1Args, Prefix + dir);
     }
     return;
   }
 
   // Add include directories specific to the selected multilib set and multilib.
   if (GCCInstallation.isValid()) {
     const MultilibSet::IncludeDirsFunc &Callback =
         Multilibs.includeDirsCallback();
     if (Callback) {
       for (const auto &Path : Callback(GCCInstallation.getMultilib()))
         addExternCSystemIncludeIfExists(
             DriverArgs, CC1Args, GCCInstallation.getInstallPath() + Path);
     }
   }
 
   addExternCSystemInclude(DriverArgs, CC1Args, D.SysRoot + "/usr/include");
 }
 
 void Solaris::addLibStdCxxIncludePaths(
     const llvm::opt::ArgList &DriverArgs,
     llvm::opt::ArgStringList &CC1Args) const {
   // We need a detected GCC installation on Solaris (similar to Linux)
   // to provide libstdc++'s headers.
   if (!GCCInstallation.isValid())
     return;
 
   // By default, look for the C++ headers in an include directory adjacent to
   // the lib directory of the GCC installation.
   // On Solaris this usually looks like /usr/gcc/X.Y/include/c++/X.Y.Z
   StringRef LibDir = GCCInstallation.getParentLibPath();
   StringRef TripleStr = GCCInstallation.getTriple().str();
   const Multilib &Multilib = GCCInstallation.getMultilib();
   const GCCVersion &Version = GCCInstallation.getVersion();
 
   // The primary search for libstdc++ supports multiarch variants.
   addLibStdCXXIncludePaths(LibDir.str() + "/../include/c++/" + Version.Text,
                            TripleStr, Multilib.includeSuffix(), DriverArgs,
                            CC1Args);
 }
diff --git a/contrib/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl_s390x.S b/contrib/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl_s390x.S
index fcff35fbc7e0..2f445e8f1b20 100644
--- a/contrib/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl_s390x.S
+++ b/contrib/llvm-project/compiler-rt/lib/tsan/rtl/tsan_rtl_s390x.S
@@ -1,47 +1,49 @@
 #include "sanitizer_common/sanitizer_asm.h"
 
 #define CFA_OFFSET 160
 #define R2_REL_OFFSET 16
 #define R3_REL_OFFSET 24
 #define R14_REL_OFFSET 112
 #define R15_REL_OFFSET 120
 #define FRAME_SIZE 160
 
 .text
 
 ASM_HIDDEN(__tsan_setjmp)
 
 .macro intercept symbol, real
 .comm \real, 8, 8
 .globl ASM_SYMBOL_INTERCEPTOR(\symbol)
 ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(\symbol))
 ASM_SYMBOL_INTERCEPTOR(\symbol):
   CFI_STARTPROC
   stmg %r2, %r3, R2_REL_OFFSET(%r15)
   CFI_REL_OFFSET(%r2, R2_REL_OFFSET)
   CFI_REL_OFFSET(%r3, R3_REL_OFFSET)
   stmg %r14, %r15, R14_REL_OFFSET(%r15)
   CFI_REL_OFFSET(%r14, R14_REL_OFFSET)
   CFI_REL_OFFSET(%r15, R15_REL_OFFSET)
   aghi %r15, -FRAME_SIZE
   CFI_ADJUST_CFA_OFFSET(FRAME_SIZE)
   la %r2, FRAME_SIZE(%r15)
   brasl %r14, ASM_SYMBOL(__tsan_setjmp)
   lmg %r14, %r15, FRAME_SIZE + R14_REL_OFFSET(%r15)
   CFI_RESTORE(%r14)
   CFI_RESTORE(%r15)
   CFI_DEF_CFA_OFFSET(CFA_OFFSET)
   lmg %r2, %r3, R2_REL_OFFSET(%r15)
   CFI_RESTORE(%r2)
   CFI_RESTORE(%r3)
   larl %r1, \real
   lg %r1, 0(%r1)
   br %r1
   CFI_ENDPROC
   ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(\symbol))
 .endm
 
 intercept setjmp, _ZN14__interception11real_setjmpE
 intercept _setjmp, _ZN14__interception12real__setjmpE
 intercept sigsetjmp, _ZN14__interception14real_sigsetjmpE
 intercept __sigsetjmp, _ZN14__interception16real___sigsetjmpE
+
+NO_EXEC_STACK_DIRECTIVE
diff --git a/contrib/llvm-project/libcxx/include/__ranges/concepts.h b/contrib/llvm-project/libcxx/include/__ranges/concepts.h
index 5f1fa834d409..e16343591cda 100644
--- a/contrib/llvm-project/libcxx/include/__ranges/concepts.h
+++ b/contrib/llvm-project/libcxx/include/__ranges/concepts.h
@@ -1,146 +1,142 @@
 // -*- C++ -*-
 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef _LIBCPP___RANGES_CONCEPTS_H
 #define _LIBCPP___RANGES_CONCEPTS_H
 
 #include <__concepts/constructible.h>
 #include <__concepts/movable.h>
 #include <__concepts/same_as.h>
 #include <__config>
 #include <__iterator/concepts.h>
 #include <__iterator/incrementable_traits.h>
 #include <__iterator/iter_move.h>
 #include <__iterator/iterator_traits.h>
 #include <__iterator/readable_traits.h>
 #include <__ranges/access.h>
 #include <__ranges/data.h>
 #include <__ranges/enable_borrowed_range.h>
 #include <__ranges/enable_view.h>
 #include <__ranges/size.h>
 #include <initializer_list>
 #include <type_traits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #pragma GCC system_header
 #endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if !defined(_LIBCPP_HAS_NO_CONCEPTS)
 
 namespace ranges {
 
   // [range.range]
 
   template <class _Tp>
   concept range = requires(_Tp& __t) {
     ranges::begin(__t); // sometimes equality-preserving
     ranges::end(__t);
   };
 
   template <class _Tp>
   concept input_range = range<_Tp> && input_iterator<iterator_t<_Tp>>;
 
   template<class _Range>
   concept borrowed_range = range<_Range> &&
     (is_lvalue_reference_v<_Range> || enable_borrowed_range<remove_cvref_t<_Range>>);
 
   // `iterator_t` defined in <__ranges/access.h>
 
   template <range _Rp>
   using sentinel_t = decltype(ranges::end(declval<_Rp&>()));
 
   template <range _Rp>
   using range_difference_t = iter_difference_t<iterator_t<_Rp>>;
 
   template <range _Rp>
   using range_value_t = iter_value_t<iterator_t<_Rp>>;
 
   template <range _Rp>
   using range_reference_t = iter_reference_t<iterator_t<_Rp>>;
 
   template <range _Rp>
   using range_rvalue_reference_t = iter_rvalue_reference_t<iterator_t<_Rp>>;
 
-#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES)
-
   // [range.sized]
   template <class _Tp>
   concept sized_range = range<_Tp> && requires(_Tp& __t) { ranges::size(__t); };
 
   template<sized_range _Rp>
   using range_size_t = decltype(ranges::size(declval<_Rp&>()));
 
   // `disable_sized_range` defined in `<__ranges/size.h>`
 
   // [range.view], views
 
   // `enable_view` defined in <__ranges/enable_view.h>
   // `view_base` defined in <__ranges/enable_view.h>
 
   template <class _Tp>
   concept view =
     range<_Tp> &&
     movable<_Tp> &&
     enable_view<_Tp>;
 
   template <class _Range>
   concept __simple_view =
     view<_Range> && range<const _Range> &&
     same_as<iterator_t<_Range>, iterator_t<const _Range>> &&
     same_as<sentinel_t<_Range>, sentinel_t<const _Range>>;
 
   // [range.refinements], other range refinements
   template <class _Rp, class _Tp>
   concept output_range = range<_Rp> && output_iterator<iterator_t<_Rp>, _Tp>;
 
   template <class _Tp>
   concept forward_range = input_range<_Tp> && forward_iterator<iterator_t<_Tp>>;
 
   template <class _Tp>
   concept bidirectional_range = forward_range<_Tp> && bidirectional_iterator<iterator_t<_Tp>>;
 
   template <class _Tp>
   concept random_access_range =
       bidirectional_range<_Tp> && random_access_iterator<iterator_t<_Tp>>;
 
   template<class _Tp>
   concept contiguous_range =
     random_access_range<_Tp> &&
     contiguous_iterator<iterator_t<_Tp>> &&
     requires(_Tp& __t) {
       { ranges::data(__t) } -> same_as<add_pointer_t<range_reference_t<_Tp>>>;
     };
 
   template <class _Tp>
   concept common_range = range<_Tp> && same_as<iterator_t<_Tp>, sentinel_t<_Tp>>;
 
   template <class _Tp>
   inline constexpr bool __is_std_initializer_list = false;
 
   template <class _Ep>
   inline constexpr bool __is_std_initializer_list<initializer_list<_Ep>> = true;
 
   template <class _Tp>
   concept viewable_range =
     range<_Tp> &&
     ((view<remove_cvref_t<_Tp>> && constructible_from<remove_cvref_t<_Tp>, _Tp>) ||
      (!view<remove_cvref_t<_Tp>> &&
       (is_lvalue_reference_v<_Tp> ||
        (movable<remove_reference_t<_Tp>> && !__is_std_initializer_list<remove_cvref_t<_Tp>>))));
 
-#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES)
-
 } // namespace ranges
 
 #endif // !defined(_LIBCPP_HAS_NO_CONCEPTS)
 
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___RANGES_CONCEPTS_H
diff --git a/contrib/llvm-project/libcxx/include/__ranges/data.h b/contrib/llvm-project/libcxx/include/__ranges/data.h
index f8d92cbc7520..f97ec8033297 100644
--- a/contrib/llvm-project/libcxx/include/__ranges/data.h
+++ b/contrib/llvm-project/libcxx/include/__ranges/data.h
@@ -1,106 +1,106 @@
 // -*- C++ -*-
 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef _LIBCPP___RANGES_DATA_H
 #define _LIBCPP___RANGES_DATA_H
 
 #include <__concepts/class_or_enum.h>
 #include <__config>
 #include <__iterator/concepts.h>
 #include <__iterator/iterator_traits.h>
 #include <__memory/pointer_traits.h>
 #include <__ranges/access.h>
 #include <__utility/auto_cast.h>
 #include <type_traits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #pragma GCC system_header
 #endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES)
+#if !defined(_LIBCPP_HAS_NO_CONCEPTS)
 
 // [range.prim.data]
 
 namespace ranges {
 namespace __data {
   template <class _Tp>
   concept __ptr_to_object = is_pointer_v<_Tp> && is_object_v<remove_pointer_t<_Tp>>;
 
   template <class _Tp>
   concept __member_data =
     __can_borrow<_Tp> &&
     __workaround_52970<_Tp> &&
     requires(_Tp&& __t) {
       { _LIBCPP_AUTO_CAST(__t.data()) } -> __ptr_to_object;
     };
 
   template <class _Tp>
   concept __ranges_begin_invocable =
     !__member_data<_Tp> &&
     __can_borrow<_Tp> &&
     requires(_Tp&& __t) {
       { ranges::begin(__t) } -> contiguous_iterator;
     };
 
   struct __fn {
     template <__member_data _Tp>
     _LIBCPP_HIDE_FROM_ABI
     constexpr auto operator()(_Tp&& __t) const
         noexcept(noexcept(__t.data())) {
       return __t.data();
     }
 
     template<__ranges_begin_invocable _Tp>
     _LIBCPP_HIDE_FROM_ABI
     constexpr auto operator()(_Tp&& __t) const
         noexcept(noexcept(_VSTD::to_address(ranges::begin(__t)))) {
       return _VSTD::to_address(ranges::begin(__t));
     }
   };
 } // namespace __data
 
 inline namespace __cpo {
   inline constexpr auto data = __data::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
 // [range.prim.cdata]
 
 namespace ranges {
 namespace __cdata {
   struct __fn {
     template <class _Tp>
       requires is_lvalue_reference_v<_Tp&&>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI
     constexpr auto operator()(_Tp&& __t) const
       noexcept(noexcept(ranges::data(static_cast<const remove_reference_t<_Tp>&>(__t))))
       -> decltype(      ranges::data(static_cast<const remove_reference_t<_Tp>&>(__t)))
       { return          ranges::data(static_cast<const remove_reference_t<_Tp>&>(__t)); }
 
     template <class _Tp>
       requires is_rvalue_reference_v<_Tp&&>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI
     constexpr auto operator()(_Tp&& __t) const
       noexcept(noexcept(ranges::data(static_cast<const _Tp&&>(__t))))
       -> decltype(      ranges::data(static_cast<const _Tp&&>(__t)))
       { return          ranges::data(static_cast<const _Tp&&>(__t)); }
   };
 } // namespace __cdata
 
 inline namespace __cpo {
   inline constexpr auto cdata = __cdata::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
-#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES)
+#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS)
 
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___RANGES_DATA_H
diff --git a/contrib/llvm-project/libcxx/include/__ranges/size.h b/contrib/llvm-project/libcxx/include/__ranges/size.h
index 2b71c03fb399..e1aaf7eba898 100644
--- a/contrib/llvm-project/libcxx/include/__ranges/size.h
+++ b/contrib/llvm-project/libcxx/include/__ranges/size.h
@@ -1,135 +1,135 @@
 // -*- C++ -*-
 //===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef _LIBCPP___RANGES_SIZE_H
 #define _LIBCPP___RANGES_SIZE_H
 
 #include <__concepts/class_or_enum.h>
 #include <__config>
 #include <__iterator/concepts.h>
 #include <__iterator/iterator_traits.h>
 #include <__ranges/access.h>
 #include <__utility/auto_cast.h>
 #include <concepts>
 #include <type_traits>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #pragma GCC system_header
 #endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES)
+#if !defined(_LIBCPP_HAS_NO_CONCEPTS)
 
 namespace ranges {
   template<class>
   inline constexpr bool disable_sized_range = false;
 } // namespace ranges
 
 // [range.prim.size]
 
 namespace ranges {
 namespace __size {
   void size(auto&) = delete;
   void size(const auto&) = delete;
 
   template <class _Tp>
   concept __size_enabled = !disable_sized_range<remove_cvref_t<_Tp>>;
 
   template <class _Tp>
   concept __member_size =
     __size_enabled<_Tp> &&
     __workaround_52970<_Tp> &&
     requires(_Tp&& __t) {
       { _LIBCPP_AUTO_CAST(__t.size()) } -> __integer_like;
     };
 
   template <class _Tp>
   concept __unqualified_size =
     __size_enabled<_Tp> &&
     !__member_size<_Tp> &&
     __class_or_enum<remove_cvref_t<_Tp>> &&
     requires(_Tp&& __t) {
       { _LIBCPP_AUTO_CAST(size(__t)) } -> __integer_like;
     };
 
   template <class _Tp>
   concept __difference =
     !__member_size<_Tp> &&
     !__unqualified_size<_Tp> &&
     __class_or_enum<remove_cvref_t<_Tp>> &&
     requires(_Tp&& __t) {
       { ranges::begin(__t) } -> forward_iterator;
       { ranges::end(__t) } -> sized_sentinel_for<decltype(ranges::begin(declval<_Tp>()))>;
     };
 
   struct __fn {
     template <class _Tp, size_t _Sz>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr size_t operator()(_Tp (&&)[_Sz]) const noexcept {
       return _Sz;
     }
 
     template <class _Tp, size_t _Sz>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr size_t operator()(_Tp (&)[_Sz]) const noexcept {
       return _Sz;
     }
 
     template <__member_size _Tp>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __integer_like auto operator()(_Tp&& __t) const
         noexcept(noexcept(_LIBCPP_AUTO_CAST(__t.size()))) {
       return _LIBCPP_AUTO_CAST(__t.size());
     }
 
     template <__unqualified_size _Tp>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __integer_like auto operator()(_Tp&& __t) const
         noexcept(noexcept(_LIBCPP_AUTO_CAST(size(__t)))) {
       return _LIBCPP_AUTO_CAST(size(__t));
     }
 
     template<__difference _Tp>
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __integer_like auto operator()(_Tp&& __t) const
         noexcept(noexcept(ranges::end(__t) - ranges::begin(__t))) {
       return _VSTD::__to_unsigned_like(ranges::end(__t) - ranges::begin(__t));
     }
   };
 } // namespace __size
 
 inline namespace __cpo {
   inline constexpr auto size = __size::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
 // [range.prim.ssize]
 
 namespace ranges {
 namespace __ssize {
   struct __fn {
     template<class _Tp>
       requires requires (_Tp&& __t) { ranges::size(__t); }
     [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr integral auto operator()(_Tp&& __t) const
       noexcept(noexcept(ranges::size(__t)))
     {
       using _Signed = make_signed_t<decltype(ranges::size(__t))>;
       if constexpr (sizeof(ptrdiff_t) > sizeof(_Signed))
         return static_cast<ptrdiff_t>(ranges::size(__t));
       else
         return static_cast<_Signed>(ranges::size(__t));
     }
   };
 } // namespace __ssize
 
 inline namespace __cpo {
   inline constexpr auto ssize = __ssize::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
-#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS) && !defined(_LIBCPP_HAS_NO_INCOMPLETE_RANGES)
+#endif // !defined(_LIBCPP_HAS_NO_CONCEPTS)
 
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP___RANGES_SIZE_H
diff --git a/contrib/llvm-project/lld/ELF/Arch/AArch64.cpp b/contrib/llvm-project/lld/ELF/Arch/AArch64.cpp
index add819ff5a80..193956fa9991 100644
--- a/contrib/llvm-project/lld/ELF/Arch/AArch64.cpp
+++ b/contrib/llvm-project/lld/ELF/Arch/AArch64.cpp
@@ -1,879 +1,881 @@
 //===- AArch64.cpp --------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "Symbols.h"
 #include "SyntheticSections.h"
 #include "Target.h"
 #include "Thunks.h"
 #include "lld/Common/ErrorHandler.h"
 #include "llvm/Object/ELF.h"
 #include "llvm/Support/Endian.h"
 
 using namespace llvm;
 using namespace llvm::support::endian;
 using namespace llvm::ELF;
 using namespace lld;
 using namespace lld::elf;
 
 // Page(Expr) is the page address of the expression Expr, defined
 // as (Expr & ~0xFFF). (This applies even if the machine page size
 // supported by the platform has a different value.)
 uint64_t elf::getAArch64Page(uint64_t expr) {
   return expr & ~static_cast<uint64_t>(0xFFF);
 }
 
 namespace {
 class AArch64 : public TargetInfo {
 public:
   AArch64();
   RelExpr getRelExpr(RelType type, const Symbol &s,
                      const uint8_t *loc) const override;
   RelType getDynRel(RelType type) const override;
   int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override;
   void writeGotPlt(uint8_t *buf, const Symbol &s) const override;
   void writePltHeader(uint8_t *buf) const override;
   void writePlt(uint8_t *buf, const Symbol &sym,
                 uint64_t pltEntryAddr) const override;
   bool needsThunk(RelExpr expr, RelType type, const InputFile *file,
                   uint64_t branchAddr, const Symbol &s,
                   int64_t a) const override;
   uint32_t getThunkSectionSpacing() const override;
   bool inBranchRange(RelType type, uint64_t src, uint64_t dst) const override;
   bool usesOnlyLowPageBits(RelType type) const override;
   void relocate(uint8_t *loc, const Relocation &rel,
                 uint64_t val) const override;
   RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override;
   void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel,
                       uint64_t val) const override;
   void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel,
                       uint64_t val) const override;
   void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
                       uint64_t val) const override;
 };
 } // namespace
 
 AArch64::AArch64() {
   copyRel = R_AARCH64_COPY;
   relativeRel = R_AARCH64_RELATIVE;
   iRelativeRel = R_AARCH64_IRELATIVE;
   gotRel = R_AARCH64_GLOB_DAT;
   pltRel = R_AARCH64_JUMP_SLOT;
   symbolicRel = R_AARCH64_ABS64;
   tlsDescRel = R_AARCH64_TLSDESC;
   tlsGotRel = R_AARCH64_TLS_TPREL64;
   pltHeaderSize = 32;
   pltEntrySize = 16;
   ipltEntrySize = 16;
   defaultMaxPageSize = 65536;
 
   // Align to the 2 MiB page size (known as a superpage or huge page).
   // FreeBSD automatically promotes 2 MiB-aligned allocations.
   defaultImageBase = 0x200000;
 
   needsThunks = true;
 }
 
 RelExpr AArch64::getRelExpr(RelType type, const Symbol &s,
                             const uint8_t *loc) const {
   switch (type) {
   case R_AARCH64_ABS16:
   case R_AARCH64_ABS32:
   case R_AARCH64_ABS64:
   case R_AARCH64_ADD_ABS_LO12_NC:
   case R_AARCH64_LDST128_ABS_LO12_NC:
   case R_AARCH64_LDST16_ABS_LO12_NC:
   case R_AARCH64_LDST32_ABS_LO12_NC:
   case R_AARCH64_LDST64_ABS_LO12_NC:
   case R_AARCH64_LDST8_ABS_LO12_NC:
   case R_AARCH64_MOVW_SABS_G0:
   case R_AARCH64_MOVW_SABS_G1:
   case R_AARCH64_MOVW_SABS_G2:
   case R_AARCH64_MOVW_UABS_G0:
   case R_AARCH64_MOVW_UABS_G0_NC:
   case R_AARCH64_MOVW_UABS_G1:
   case R_AARCH64_MOVW_UABS_G1_NC:
   case R_AARCH64_MOVW_UABS_G2:
   case R_AARCH64_MOVW_UABS_G2_NC:
   case R_AARCH64_MOVW_UABS_G3:
     return R_ABS;
   case R_AARCH64_TLSDESC_ADR_PAGE21:
     return R_AARCH64_TLSDESC_PAGE;
   case R_AARCH64_TLSDESC_LD64_LO12:
   case R_AARCH64_TLSDESC_ADD_LO12:
     return R_TLSDESC;
   case R_AARCH64_TLSDESC_CALL:
     return R_TLSDESC_CALL;
   case R_AARCH64_TLSLE_ADD_TPREL_HI12:
   case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
   case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC:
   case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC:
   case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC:
   case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC:
   case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC:
   case R_AARCH64_TLSLE_MOVW_TPREL_G0:
   case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC:
   case R_AARCH64_TLSLE_MOVW_TPREL_G1:
   case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC:
   case R_AARCH64_TLSLE_MOVW_TPREL_G2:
     return R_TPREL;
   case R_AARCH64_CALL26:
   case R_AARCH64_CONDBR19:
   case R_AARCH64_JUMP26:
   case R_AARCH64_TSTBR14:
   case R_AARCH64_PLT32:
     return R_PLT_PC;
   case R_AARCH64_PREL16:
   case R_AARCH64_PREL32:
   case R_AARCH64_PREL64:
   case R_AARCH64_ADR_PREL_LO21:
   case R_AARCH64_LD_PREL_LO19:
   case R_AARCH64_MOVW_PREL_G0:
   case R_AARCH64_MOVW_PREL_G0_NC:
   case R_AARCH64_MOVW_PREL_G1:
   case R_AARCH64_MOVW_PREL_G1_NC:
   case R_AARCH64_MOVW_PREL_G2:
   case R_AARCH64_MOVW_PREL_G2_NC:
   case R_AARCH64_MOVW_PREL_G3:
     return R_PC;
   case R_AARCH64_ADR_PREL_PG_HI21:
   case R_AARCH64_ADR_PREL_PG_HI21_NC:
     return R_AARCH64_PAGE_PC;
   case R_AARCH64_LD64_GOT_LO12_NC:
   case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
     return R_GOT;
   case R_AARCH64_LD64_GOTPAGE_LO15:
     return R_AARCH64_GOT_PAGE;
   case R_AARCH64_ADR_GOT_PAGE:
   case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
     return R_AARCH64_GOT_PAGE_PC;
   case R_AARCH64_NONE:
     return R_NONE;
   default:
     error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) +
           ") against symbol " + toString(s));
     return R_NONE;
   }
 }
 
 RelExpr AArch64::adjustTlsExpr(RelType type, RelExpr expr) const {
   if (expr == R_RELAX_TLS_GD_TO_IE) {
     if (type == R_AARCH64_TLSDESC_ADR_PAGE21)
       return R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC;
     return R_RELAX_TLS_GD_TO_IE_ABS;
   }
   return expr;
 }
 
 bool AArch64::usesOnlyLowPageBits(RelType type) const {
   switch (type) {
   default:
     return false;
   case R_AARCH64_ADD_ABS_LO12_NC:
   case R_AARCH64_LD64_GOT_LO12_NC:
   case R_AARCH64_LDST128_ABS_LO12_NC:
   case R_AARCH64_LDST16_ABS_LO12_NC:
   case R_AARCH64_LDST32_ABS_LO12_NC:
   case R_AARCH64_LDST64_ABS_LO12_NC:
   case R_AARCH64_LDST8_ABS_LO12_NC:
   case R_AARCH64_TLSDESC_ADD_LO12:
   case R_AARCH64_TLSDESC_LD64_LO12:
   case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
     return true;
   }
 }
 
 RelType AArch64::getDynRel(RelType type) const {
   if (type == R_AARCH64_ABS64)
     return type;
   return R_AARCH64_NONE;
 }
 
 int64_t AArch64::getImplicitAddend(const uint8_t *buf, RelType type) const {
   switch (type) {
   case R_AARCH64_TLSDESC:
     return read64(buf + 8);
   case R_AARCH64_NONE:
     return 0;
   case R_AARCH64_PREL32:
     return SignExtend64<32>(read32(buf));
   case R_AARCH64_ABS64:
   case R_AARCH64_PREL64:
     return read64(buf);
   default:
     internalLinkerError(getErrorLocation(buf),
                         "cannot read addend for relocation " + toString(type));
     return 0;
   }
 }
 
 void AArch64::writeGotPlt(uint8_t *buf, const Symbol &) const {
   write64(buf, in.plt->getVA());
 }
 
 void AArch64::writePltHeader(uint8_t *buf) const {
   const uint8_t pltData[] = {
       0xf0, 0x7b, 0xbf, 0xa9, // stp    x16, x30, [sp,#-16]!
       0x10, 0x00, 0x00, 0x90, // adrp   x16, Page(&(.plt.got[2]))
       0x11, 0x02, 0x40, 0xf9, // ldr    x17, [x16, Offset(&(.plt.got[2]))]
       0x10, 0x02, 0x00, 0x91, // add    x16, x16, Offset(&(.plt.got[2]))
       0x20, 0x02, 0x1f, 0xd6, // br     x17
       0x1f, 0x20, 0x03, 0xd5, // nop
       0x1f, 0x20, 0x03, 0xd5, // nop
       0x1f, 0x20, 0x03, 0xd5  // nop
   };
   memcpy(buf, pltData, sizeof(pltData));
 
   uint64_t got = in.gotPlt->getVA();
   uint64_t plt = in.plt->getVA();
   relocateNoSym(buf + 4, R_AARCH64_ADR_PREL_PG_HI21,
                 getAArch64Page(got + 16) - getAArch64Page(plt + 4));
   relocateNoSym(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16);
   relocateNoSym(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16);
 }
 
 void AArch64::writePlt(uint8_t *buf, const Symbol &sym,
                        uint64_t pltEntryAddr) const {
   const uint8_t inst[] = {
       0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n]))
       0x11, 0x02, 0x40, 0xf9, // ldr  x17, [x16, Offset(&(.plt.got[n]))]
       0x10, 0x02, 0x00, 0x91, // add  x16, x16, Offset(&(.plt.got[n]))
       0x20, 0x02, 0x1f, 0xd6  // br   x17
   };
   memcpy(buf, inst, sizeof(inst));
 
   uint64_t gotPltEntryAddr = sym.getGotPltVA();
   relocateNoSym(buf, R_AARCH64_ADR_PREL_PG_HI21,
                 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr));
   relocateNoSym(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr);
   relocateNoSym(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr);
 }
 
 bool AArch64::needsThunk(RelExpr expr, RelType type, const InputFile *file,
                          uint64_t branchAddr, const Symbol &s,
                          int64_t a) const {
-  // If s is an undefined weak symbol and does not have a PLT entry then it
-  // will be resolved as a branch to the next instruction.
-  if (s.isUndefWeak() && !s.isInPlt())
+  // If s is an undefined weak symbol and does not have a PLT entry then it will
+  // be resolved as a branch to the next instruction. If it is hidden, its
+  // binding has been converted to local, so we just check isUndefined() here. A
+  // undefined non-weak symbol will have been errored.
+  if (s.isUndefined() && !s.isInPlt())
     return false;
   // ELF for the ARM 64-bit architecture, section Call and Jump relocations
   // only permits range extension thunks for R_AARCH64_CALL26 and
   // R_AARCH64_JUMP26 relocation types.
   if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 &&
       type != R_AARCH64_PLT32)
     return false;
   uint64_t dst = expr == R_PLT_PC ? s.getPltVA() : s.getVA(a);
   return !inBranchRange(type, branchAddr, dst);
 }
 
 uint32_t AArch64::getThunkSectionSpacing() const {
   // See comment in Arch/ARM.cpp for a more detailed explanation of
   // getThunkSectionSpacing(). For AArch64 the only branches we are permitted to
   // Thunk have a range of +/- 128 MiB
   return (128 * 1024 * 1024) - 0x30000;
 }
 
 bool AArch64::inBranchRange(RelType type, uint64_t src, uint64_t dst) const {
   if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 &&
       type != R_AARCH64_PLT32)
     return true;
   // The AArch64 call and unconditional branch instructions have a range of
   // +/- 128 MiB. The PLT32 relocation supports a range up to +/- 2 GiB.
   uint64_t range =
       type == R_AARCH64_PLT32 ? (UINT64_C(1) << 31) : (128 * 1024 * 1024);
   if (dst > src) {
     // Immediate of branch is signed.
     range -= 4;
     return dst - src <= range;
   }
   return src - dst <= range;
 }
 
 static void write32AArch64Addr(uint8_t *l, uint64_t imm) {
   uint32_t immLo = (imm & 0x3) << 29;
   uint32_t immHi = (imm & 0x1FFFFC) << 3;
   uint64_t mask = (0x3 << 29) | (0x1FFFFC << 3);
   write32le(l, (read32le(l) & ~mask) | immLo | immHi);
 }
 
 // Return the bits [Start, End] from Val shifted Start bits.
 // For instance, getBits(0xF0, 4, 8) returns 0xF.
 static uint64_t getBits(uint64_t val, int start, int end) {
   uint64_t mask = ((uint64_t)1 << (end + 1 - start)) - 1;
   return (val >> start) & mask;
 }
 
 static void or32le(uint8_t *p, int32_t v) { write32le(p, read32le(p) | v); }
 
 // Update the immediate field in a AARCH64 ldr, str, and add instruction.
 static void or32AArch64Imm(uint8_t *l, uint64_t imm) {
   or32le(l, (imm & 0xFFF) << 10);
 }
 
 // Update the immediate field in an AArch64 movk, movn or movz instruction
 // for a signed relocation, and update the opcode of a movn or movz instruction
 // to match the sign of the operand.
 static void writeSMovWImm(uint8_t *loc, uint32_t imm) {
   uint32_t inst = read32le(loc);
   // Opcode field is bits 30, 29, with 10 = movz, 00 = movn and 11 = movk.
   if (!(inst & (1 << 29))) {
     // movn or movz.
     if (imm & 0x10000) {
       // Change opcode to movn, which takes an inverted operand.
       imm ^= 0xFFFF;
       inst &= ~(1 << 30);
     } else {
       // Change opcode to movz.
       inst |= 1 << 30;
     }
   }
   write32le(loc, inst | ((imm & 0xFFFF) << 5));
 }
 
 void AArch64::relocate(uint8_t *loc, const Relocation &rel,
                        uint64_t val) const {
   switch (rel.type) {
   case R_AARCH64_ABS16:
   case R_AARCH64_PREL16:
     checkIntUInt(loc, val, 16, rel);
     write16(loc, val);
     break;
   case R_AARCH64_ABS32:
   case R_AARCH64_PREL32:
     checkIntUInt(loc, val, 32, rel);
     write32(loc, val);
     break;
   case R_AARCH64_PLT32:
     checkInt(loc, val, 32, rel);
     write32(loc, val);
     break;
   case R_AARCH64_ABS64:
   case R_AARCH64_PREL64:
     write64(loc, val);
     break;
   case R_AARCH64_ADD_ABS_LO12_NC:
     or32AArch64Imm(loc, val);
     break;
   case R_AARCH64_ADR_GOT_PAGE:
   case R_AARCH64_ADR_PREL_PG_HI21:
   case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
   case R_AARCH64_TLSDESC_ADR_PAGE21:
     checkInt(loc, val, 33, rel);
     LLVM_FALLTHROUGH;
   case R_AARCH64_ADR_PREL_PG_HI21_NC:
     write32AArch64Addr(loc, val >> 12);
     break;
   case R_AARCH64_ADR_PREL_LO21:
     checkInt(loc, val, 21, rel);
     write32AArch64Addr(loc, val);
     break;
   case R_AARCH64_JUMP26:
     // Normally we would just write the bits of the immediate field, however
     // when patching instructions for the cpu errata fix -fix-cortex-a53-843419
     // we want to replace a non-branch instruction with a branch immediate
     // instruction. By writing all the bits of the instruction including the
     // opcode and the immediate (0 001 | 01 imm26) we can do this
     // transformation by placing a R_AARCH64_JUMP26 relocation at the offset of
     // the instruction we want to patch.
     write32le(loc, 0x14000000);
     LLVM_FALLTHROUGH;
   case R_AARCH64_CALL26:
     checkInt(loc, val, 28, rel);
     or32le(loc, (val & 0x0FFFFFFC) >> 2);
     break;
   case R_AARCH64_CONDBR19:
   case R_AARCH64_LD_PREL_LO19:
     checkAlignment(loc, val, 4, rel);
     checkInt(loc, val, 21, rel);
     or32le(loc, (val & 0x1FFFFC) << 3);
     break;
   case R_AARCH64_LDST8_ABS_LO12_NC:
   case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC:
     or32AArch64Imm(loc, getBits(val, 0, 11));
     break;
   case R_AARCH64_LDST16_ABS_LO12_NC:
   case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC:
     checkAlignment(loc, val, 2, rel);
     or32AArch64Imm(loc, getBits(val, 1, 11));
     break;
   case R_AARCH64_LDST32_ABS_LO12_NC:
   case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC:
     checkAlignment(loc, val, 4, rel);
     or32AArch64Imm(loc, getBits(val, 2, 11));
     break;
   case R_AARCH64_LDST64_ABS_LO12_NC:
   case R_AARCH64_LD64_GOT_LO12_NC:
   case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
   case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC:
   case R_AARCH64_TLSDESC_LD64_LO12:
     checkAlignment(loc, val, 8, rel);
     or32AArch64Imm(loc, getBits(val, 3, 11));
     break;
   case R_AARCH64_LDST128_ABS_LO12_NC:
   case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC:
     checkAlignment(loc, val, 16, rel);
     or32AArch64Imm(loc, getBits(val, 4, 11));
     break;
   case R_AARCH64_LD64_GOTPAGE_LO15:
     checkAlignment(loc, val, 8, rel);
     or32AArch64Imm(loc, getBits(val, 3, 14));
     break;
   case R_AARCH64_MOVW_UABS_G0:
     checkUInt(loc, val, 16, rel);
     LLVM_FALLTHROUGH;
   case R_AARCH64_MOVW_UABS_G0_NC:
     or32le(loc, (val & 0xFFFF) << 5);
     break;
   case R_AARCH64_MOVW_UABS_G1:
     checkUInt(loc, val, 32, rel);
     LLVM_FALLTHROUGH;
   case R_AARCH64_MOVW_UABS_G1_NC:
     or32le(loc, (val & 0xFFFF0000) >> 11);
     break;
   case R_AARCH64_MOVW_UABS_G2:
     checkUInt(loc, val, 48, rel);
     LLVM_FALLTHROUGH;
   case R_AARCH64_MOVW_UABS_G2_NC:
     or32le(loc, (val & 0xFFFF00000000) >> 27);
     break;
   case R_AARCH64_MOVW_UABS_G3:
     or32le(loc, (val & 0xFFFF000000000000) >> 43);
     break;
   case R_AARCH64_MOVW_PREL_G0:
   case R_AARCH64_MOVW_SABS_G0:
   case R_AARCH64_TLSLE_MOVW_TPREL_G0:
     checkInt(loc, val, 17, rel);
     LLVM_FALLTHROUGH;
   case R_AARCH64_MOVW_PREL_G0_NC:
   case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC:
     writeSMovWImm(loc, val);
     break;
   case R_AARCH64_MOVW_PREL_G1:
   case R_AARCH64_MOVW_SABS_G1:
   case R_AARCH64_TLSLE_MOVW_TPREL_G1:
     checkInt(loc, val, 33, rel);
     LLVM_FALLTHROUGH;
   case R_AARCH64_MOVW_PREL_G1_NC:
   case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC:
     writeSMovWImm(loc, val >> 16);
     break;
   case R_AARCH64_MOVW_PREL_G2:
   case R_AARCH64_MOVW_SABS_G2:
   case R_AARCH64_TLSLE_MOVW_TPREL_G2:
     checkInt(loc, val, 49, rel);
     LLVM_FALLTHROUGH;
   case R_AARCH64_MOVW_PREL_G2_NC:
     writeSMovWImm(loc, val >> 32);
     break;
   case R_AARCH64_MOVW_PREL_G3:
     writeSMovWImm(loc, val >> 48);
     break;
   case R_AARCH64_TSTBR14:
     checkInt(loc, val, 16, rel);
     or32le(loc, (val & 0xFFFC) << 3);
     break;
   case R_AARCH64_TLSLE_ADD_TPREL_HI12:
     checkUInt(loc, val, 24, rel);
     or32AArch64Imm(loc, val >> 12);
     break;
   case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
   case R_AARCH64_TLSDESC_ADD_LO12:
     or32AArch64Imm(loc, val);
     break;
   case R_AARCH64_TLSDESC:
     // For R_AARCH64_TLSDESC the addend is stored in the second 64-bit word.
     write64(loc + 8, val);
     break;
   default:
     llvm_unreachable("unknown relocation");
   }
 }
 
 void AArch64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel,
                              uint64_t val) const {
   // TLSDESC Global-Dynamic relocation are in the form:
   //   adrp    x0, :tlsdesc:v             [R_AARCH64_TLSDESC_ADR_PAGE21]
   //   ldr     x1, [x0, #:tlsdesc_lo12:v  [R_AARCH64_TLSDESC_LD64_LO12]
   //   add     x0, x0, :tlsdesc_los:v     [R_AARCH64_TLSDESC_ADD_LO12]
   //   .tlsdesccall                       [R_AARCH64_TLSDESC_CALL]
   //   blr     x1
   // And it can optimized to:
   //   movz    x0, #0x0, lsl #16
   //   movk    x0, #0x10
   //   nop
   //   nop
   checkUInt(loc, val, 32, rel);
 
   switch (rel.type) {
   case R_AARCH64_TLSDESC_ADD_LO12:
   case R_AARCH64_TLSDESC_CALL:
     write32le(loc, 0xd503201f); // nop
     return;
   case R_AARCH64_TLSDESC_ADR_PAGE21:
     write32le(loc, 0xd2a00000 | (((val >> 16) & 0xffff) << 5)); // movz
     return;
   case R_AARCH64_TLSDESC_LD64_LO12:
     write32le(loc, 0xf2800000 | ((val & 0xffff) << 5)); // movk
     return;
   default:
     llvm_unreachable("unsupported relocation for TLS GD to LE relaxation");
   }
 }
 
 void AArch64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel,
                              uint64_t val) const {
   // TLSDESC Global-Dynamic relocation are in the form:
   //   adrp    x0, :tlsdesc:v             [R_AARCH64_TLSDESC_ADR_PAGE21]
   //   ldr     x1, [x0, #:tlsdesc_lo12:v  [R_AARCH64_TLSDESC_LD64_LO12]
   //   add     x0, x0, :tlsdesc_los:v     [R_AARCH64_TLSDESC_ADD_LO12]
   //   .tlsdesccall                       [R_AARCH64_TLSDESC_CALL]
   //   blr     x1
   // And it can optimized to:
   //   adrp    x0, :gottprel:v
   //   ldr     x0, [x0, :gottprel_lo12:v]
   //   nop
   //   nop
 
   switch (rel.type) {
   case R_AARCH64_TLSDESC_ADD_LO12:
   case R_AARCH64_TLSDESC_CALL:
     write32le(loc, 0xd503201f); // nop
     break;
   case R_AARCH64_TLSDESC_ADR_PAGE21:
     write32le(loc, 0x90000000); // adrp
     relocateNoSym(loc, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21, val);
     break;
   case R_AARCH64_TLSDESC_LD64_LO12:
     write32le(loc, 0xf9400000); // ldr
     relocateNoSym(loc, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC, val);
     break;
   default:
     llvm_unreachable("unsupported relocation for TLS GD to LE relaxation");
   }
 }
 
 void AArch64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
                              uint64_t val) const {
   checkUInt(loc, val, 32, rel);
 
   if (rel.type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) {
     // Generate MOVZ.
     uint32_t regNo = read32le(loc) & 0x1f;
     write32le(loc, (0xd2a00000 | regNo) | (((val >> 16) & 0xffff) << 5));
     return;
   }
   if (rel.type == R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) {
     // Generate MOVK.
     uint32_t regNo = read32le(loc) & 0x1f;
     write32le(loc, (0xf2800000 | regNo) | ((val & 0xffff) << 5));
     return;
   }
   llvm_unreachable("invalid relocation for TLS IE to LE relaxation");
 }
 
 AArch64Relaxer::AArch64Relaxer(ArrayRef<Relocation> relocs) {
   if (!config->relax || config->emachine != EM_AARCH64) {
     safeToRelaxAdrpLdr = false;
     return;
   }
   // Check if R_AARCH64_ADR_GOT_PAGE and R_AARCH64_LD64_GOT_LO12_NC
   // always appear in pairs.
   size_t i = 0;
   const size_t size = relocs.size();
   for (; i != size; ++i) {
     if (relocs[i].type == R_AARCH64_ADR_GOT_PAGE) {
       if (i + 1 < size && relocs[i + 1].type == R_AARCH64_LD64_GOT_LO12_NC) {
         ++i;
         continue;
       }
       break;
     } else if (relocs[i].type == R_AARCH64_LD64_GOT_LO12_NC) {
       break;
     }
   }
   safeToRelaxAdrpLdr = i == size;
 }
 
 bool AArch64Relaxer::tryRelaxAdrpAdd(const Relocation &adrpRel,
                                      const Relocation &addRel, uint64_t secAddr,
                                      uint8_t *buf) const {
   // When the address of sym is within the range of ADR then
   // we may relax
   // ADRP xn, sym
   // ADD  xn, xn, :lo12: sym
   // to
   // NOP
   // ADR xn, sym
   if (!config->relax || adrpRel.type != R_AARCH64_ADR_PREL_PG_HI21 ||
       addRel.type != R_AARCH64_ADD_ABS_LO12_NC)
     return false;
   // Check if the relocations apply to consecutive instructions.
   if (adrpRel.offset + 4 != addRel.offset)
     return false;
   if (adrpRel.sym != addRel.sym)
     return false;
   if (adrpRel.addend != 0 || addRel.addend != 0)
     return false;
 
   uint32_t adrpInstr = read32le(buf + adrpRel.offset);
   uint32_t addInstr = read32le(buf + addRel.offset);
   // Check if the first instruction is ADRP and the second instruction is ADD.
   if ((adrpInstr & 0x9f000000) != 0x90000000 ||
       (addInstr & 0xffc00000) != 0x91000000)
     return false;
   uint32_t adrpDestReg = adrpInstr & 0x1f;
   uint32_t addDestReg = addInstr & 0x1f;
   uint32_t addSrcReg = (addInstr >> 5) & 0x1f;
   if (adrpDestReg != addDestReg || adrpDestReg != addSrcReg)
     return false;
 
   Symbol &sym = *adrpRel.sym;
   // Check if the address difference is within 1MiB range.
   int64_t val = sym.getVA() - (secAddr + addRel.offset);
   if (val < -1024 * 1024 || val >= 1024 * 1024)
     return false;
 
   Relocation adrRel = {R_ABS, R_AARCH64_ADR_PREL_LO21, addRel.offset,
                        /*addend=*/0, &sym};
   // nop
   write32le(buf + adrpRel.offset, 0xd503201f);
   // adr x_<dest_reg>
   write32le(buf + adrRel.offset, 0x10000000 | adrpDestReg);
   target->relocate(buf + adrRel.offset, adrRel, val);
   return true;
 }
 
 bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel,
                                      const Relocation &ldrRel, uint64_t secAddr,
                                      uint8_t *buf) const {
   if (!safeToRelaxAdrpLdr)
     return false;
 
   // When the definition of sym is not preemptible then we may
   // be able to relax
   // ADRP xn, :got: sym
   // LDR xn, [ xn :got_lo12: sym]
   // to
   // ADRP xn, sym
   // ADD xn, xn, :lo_12: sym
 
   if (adrpRel.type != R_AARCH64_ADR_GOT_PAGE ||
       ldrRel.type != R_AARCH64_LD64_GOT_LO12_NC)
     return false;
   // Check if the relocations apply to consecutive instructions.
   if (adrpRel.offset + 4 != ldrRel.offset)
     return false;
   // Check if the relocations reference the same symbol and
   // skip undefined, preemptible and STT_GNU_IFUNC symbols.
   if (!adrpRel.sym || adrpRel.sym != ldrRel.sym || !adrpRel.sym->isDefined() ||
       adrpRel.sym->isPreemptible || adrpRel.sym->isGnuIFunc())
     return false;
   // Check if the addends of the both relocations are zero.
   if (adrpRel.addend != 0 || ldrRel.addend != 0)
     return false;
   uint32_t adrpInstr = read32le(buf + adrpRel.offset);
   uint32_t ldrInstr = read32le(buf + ldrRel.offset);
   // Check if the first instruction is ADRP and the second instruction is LDR.
   if ((adrpInstr & 0x9f000000) != 0x90000000 ||
       (ldrInstr & 0x3b000000) != 0x39000000)
     return false;
   // Check the value of the sf bit.
   if (!(ldrInstr >> 31))
     return false;
   uint32_t adrpDestReg = adrpInstr & 0x1f;
   uint32_t ldrDestReg = ldrInstr & 0x1f;
   uint32_t ldrSrcReg = (ldrInstr >> 5) & 0x1f;
   // Check if ADPR and LDR use the same register.
   if (adrpDestReg != ldrDestReg || adrpDestReg != ldrSrcReg)
     return false;
 
   Symbol &sym = *adrpRel.sym;
   // Check if the address difference is within 4GB range.
   int64_t val =
       getAArch64Page(sym.getVA()) - getAArch64Page(secAddr + adrpRel.offset);
   if (val != llvm::SignExtend64(val, 33))
     return false;
 
   Relocation adrpSymRel = {R_AARCH64_PAGE_PC, R_AARCH64_ADR_PREL_PG_HI21,
                            adrpRel.offset, /*addend=*/0, &sym};
   Relocation addRel = {R_ABS, R_AARCH64_ADD_ABS_LO12_NC, ldrRel.offset,
                        /*addend=*/0, &sym};
 
   // adrp x_<dest_reg>
   write32le(buf + adrpSymRel.offset, 0x90000000 | adrpDestReg);
   // add x_<dest reg>, x_<dest reg>
   write32le(buf + addRel.offset, 0x91000000 | adrpDestReg | (adrpDestReg << 5));
 
   target->relocate(buf + adrpSymRel.offset, adrpSymRel,
                    SignExtend64(getAArch64Page(sym.getVA()) -
                                     getAArch64Page(secAddr + adrpSymRel.offset),
                                 64));
   target->relocate(buf + addRel.offset, addRel, SignExtend64(sym.getVA(), 64));
   tryRelaxAdrpAdd(adrpSymRel, addRel, secAddr, buf);
   return true;
 }
 
 // AArch64 may use security features in variant PLT sequences. These are:
 // Pointer Authentication (PAC), introduced in armv8.3-a and Branch Target
 // Indicator (BTI) introduced in armv8.5-a. The additional instructions used
 // in the variant Plt sequences are encoded in the Hint space so they can be
 // deployed on older architectures, which treat the instructions as a nop.
 // PAC and BTI can be combined leading to the following combinations:
 // writePltHeader
 // writePltHeaderBti (no PAC Header needed)
 // writePlt
 // writePltBti (BTI only)
 // writePltPac (PAC only)
 // writePltBtiPac (BTI and PAC)
 //
 // When PAC is enabled the dynamic loader encrypts the address that it places
 // in the .got.plt using the pacia1716 instruction which encrypts the value in
 // x17 using the modifier in x16. The static linker places autia1716 before the
 // indirect branch to x17 to authenticate the address in x17 with the modifier
 // in x16. This makes it more difficult for an attacker to modify the value in
 // the .got.plt.
 //
 // When BTI is enabled all indirect branches must land on a bti instruction.
 // The static linker must place a bti instruction at the start of any PLT entry
 // that may be the target of an indirect branch. As the PLT entries call the
 // lazy resolver indirectly this must have a bti instruction at start. In
 // general a bti instruction is not needed for a PLT entry as indirect calls
 // are resolved to the function address and not the PLT entry for the function.
 // There are a small number of cases where the PLT address can escape, such as
 // taking the address of a function or ifunc via a non got-generating
 // relocation, and a shared library refers to that symbol.
 //
 // We use the bti c variant of the instruction which permits indirect branches
 // (br) via x16/x17 and indirect function calls (blr) via any register. The ABI
 // guarantees that all indirect branches from code requiring BTI protection
 // will go via x16/x17
 
 namespace {
 class AArch64BtiPac final : public AArch64 {
 public:
   AArch64BtiPac();
   void writePltHeader(uint8_t *buf) const override;
   void writePlt(uint8_t *buf, const Symbol &sym,
                 uint64_t pltEntryAddr) const override;
 
 private:
   bool btiHeader; // bti instruction needed in PLT Header and Entry
   bool pacEntry;  // autia1716 instruction needed in PLT Entry
 };
 } // namespace
 
 AArch64BtiPac::AArch64BtiPac() {
   btiHeader = (config->andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI);
   // A BTI (Branch Target Indicator) Plt Entry is only required if the
   // address of the PLT entry can be taken by the program, which permits an
   // indirect jump to the PLT entry. This can happen when the address
   // of the PLT entry for a function is canonicalised due to the address of
   // the function in an executable being taken by a shared library, or
   // non-preemptible ifunc referenced by non-GOT-generating, non-PLT-generating
   // relocations.
   // The PAC PLT entries require dynamic loader support and this isn't known
   // from properties in the objects, so we use the command line flag.
   pacEntry = config->zPacPlt;
 
   if (btiHeader || pacEntry) {
     pltEntrySize = 24;
     ipltEntrySize = 24;
   }
 }
 
 void AArch64BtiPac::writePltHeader(uint8_t *buf) const {
   const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c
   const uint8_t pltData[] = {
       0xf0, 0x7b, 0xbf, 0xa9, // stp    x16, x30, [sp,#-16]!
       0x10, 0x00, 0x00, 0x90, // adrp   x16, Page(&(.plt.got[2]))
       0x11, 0x02, 0x40, 0xf9, // ldr    x17, [x16, Offset(&(.plt.got[2]))]
       0x10, 0x02, 0x00, 0x91, // add    x16, x16, Offset(&(.plt.got[2]))
       0x20, 0x02, 0x1f, 0xd6, // br     x17
       0x1f, 0x20, 0x03, 0xd5, // nop
       0x1f, 0x20, 0x03, 0xd5  // nop
   };
   const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop
 
   uint64_t got = in.gotPlt->getVA();
   uint64_t plt = in.plt->getVA();
 
   if (btiHeader) {
     // PltHeader is called indirectly by plt[N]. Prefix pltData with a BTI C
     // instruction.
     memcpy(buf, btiData, sizeof(btiData));
     buf += sizeof(btiData);
     plt += sizeof(btiData);
   }
   memcpy(buf, pltData, sizeof(pltData));
 
   relocateNoSym(buf + 4, R_AARCH64_ADR_PREL_PG_HI21,
                 getAArch64Page(got + 16) - getAArch64Page(plt + 8));
   relocateNoSym(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16);
   relocateNoSym(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16);
   if (!btiHeader)
     // We didn't add the BTI c instruction so round out size with NOP.
     memcpy(buf + sizeof(pltData), nopData, sizeof(nopData));
 }
 
 void AArch64BtiPac::writePlt(uint8_t *buf, const Symbol &sym,
                              uint64_t pltEntryAddr) const {
   // The PLT entry is of the form:
   // [btiData] addrInst (pacBr | stdBr) [nopData]
   const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c
   const uint8_t addrInst[] = {
       0x10, 0x00, 0x00, 0x90,  // adrp x16, Page(&(.plt.got[n]))
       0x11, 0x02, 0x40, 0xf9,  // ldr  x17, [x16, Offset(&(.plt.got[n]))]
       0x10, 0x02, 0x00, 0x91   // add  x16, x16, Offset(&(.plt.got[n]))
   };
   const uint8_t pacBr[] = {
       0x9f, 0x21, 0x03, 0xd5,  // autia1716
       0x20, 0x02, 0x1f, 0xd6   // br   x17
   };
   const uint8_t stdBr[] = {
       0x20, 0x02, 0x1f, 0xd6,  // br   x17
       0x1f, 0x20, 0x03, 0xd5   // nop
   };
   const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop
 
   // needsCopy indicates a non-ifunc canonical PLT entry whose address may
   // escape to shared objects. isInIplt indicates a non-preemptible ifunc. Its
   // address may escape if referenced by a direct relocation. The condition is
   // conservative.
   bool hasBti = btiHeader && (sym.needsCopy || sym.isInIplt);
   if (hasBti) {
     memcpy(buf, btiData, sizeof(btiData));
     buf += sizeof(btiData);
     pltEntryAddr += sizeof(btiData);
   }
 
   uint64_t gotPltEntryAddr = sym.getGotPltVA();
   memcpy(buf, addrInst, sizeof(addrInst));
   relocateNoSym(buf, R_AARCH64_ADR_PREL_PG_HI21,
                 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr));
   relocateNoSym(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr);
   relocateNoSym(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr);
 
   if (pacEntry)
     memcpy(buf + sizeof(addrInst), pacBr, sizeof(pacBr));
   else
     memcpy(buf + sizeof(addrInst), stdBr, sizeof(stdBr));
   if (!hasBti)
     // We didn't add the BTI c instruction so round out size with NOP.
     memcpy(buf + sizeof(addrInst) + sizeof(stdBr), nopData, sizeof(nopData));
 }
 
 static TargetInfo *getTargetInfo() {
   if (config->andFeatures & (GNU_PROPERTY_AARCH64_FEATURE_1_BTI |
                              GNU_PROPERTY_AARCH64_FEATURE_1_PAC)) {
     static AArch64BtiPac t;
     return &t;
   }
   static AArch64 t;
   return &t;
 }
 
 TargetInfo *elf::getAArch64TargetInfo() { return getTargetInfo(); }
diff --git a/contrib/llvm-project/lld/ELF/Arch/ARM.cpp b/contrib/llvm-project/lld/ELF/Arch/ARM.cpp
index b7c2eb74757c..b6b32f0500a4 100644
--- a/contrib/llvm-project/lld/ELF/Arch/ARM.cpp
+++ b/contrib/llvm-project/lld/ELF/Arch/ARM.cpp
@@ -1,916 +1,918 @@
 //===- ARM.cpp ------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "InputFiles.h"
 #include "Symbols.h"
 #include "SyntheticSections.h"
 #include "Target.h"
 #include "Thunks.h"
 #include "lld/Common/ErrorHandler.h"
 #include "llvm/Object/ELF.h"
 #include "llvm/Support/Endian.h"
 
 using namespace llvm;
 using namespace llvm::support::endian;
 using namespace llvm::ELF;
 using namespace lld;
 using namespace lld::elf;
 
 namespace {
 class ARM final : public TargetInfo {
 public:
   ARM();
   uint32_t calcEFlags() const override;
   RelExpr getRelExpr(RelType type, const Symbol &s,
                      const uint8_t *loc) const override;
   RelType getDynRel(RelType type) const override;
   int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override;
   void writeGotPlt(uint8_t *buf, const Symbol &s) const override;
   void writeIgotPlt(uint8_t *buf, const Symbol &s) const override;
   void writePltHeader(uint8_t *buf) const override;
   void writePlt(uint8_t *buf, const Symbol &sym,
                 uint64_t pltEntryAddr) const override;
   void addPltSymbols(InputSection &isec, uint64_t off) const override;
   void addPltHeaderSymbols(InputSection &isd) const override;
   bool needsThunk(RelExpr expr, RelType type, const InputFile *file,
                   uint64_t branchAddr, const Symbol &s,
                   int64_t a) const override;
   uint32_t getThunkSectionSpacing() const override;
   bool inBranchRange(RelType type, uint64_t src, uint64_t dst) const override;
   void relocate(uint8_t *loc, const Relocation &rel,
                 uint64_t val) const override;
 };
 } // namespace
 
 ARM::ARM() {
   copyRel = R_ARM_COPY;
   relativeRel = R_ARM_RELATIVE;
   iRelativeRel = R_ARM_IRELATIVE;
   gotRel = R_ARM_GLOB_DAT;
   pltRel = R_ARM_JUMP_SLOT;
   symbolicRel = R_ARM_ABS32;
   tlsGotRel = R_ARM_TLS_TPOFF32;
   tlsModuleIndexRel = R_ARM_TLS_DTPMOD32;
   tlsOffsetRel = R_ARM_TLS_DTPOFF32;
   pltHeaderSize = 32;
   pltEntrySize = 16;
   ipltEntrySize = 16;
   trapInstr = {0xd4, 0xd4, 0xd4, 0xd4};
   needsThunks = true;
   defaultMaxPageSize = 65536;
 }
 
 uint32_t ARM::calcEFlags() const {
   // The ABIFloatType is used by loaders to detect the floating point calling
   // convention.
   uint32_t abiFloatType = 0;
   if (config->armVFPArgs == ARMVFPArgKind::Base ||
       config->armVFPArgs == ARMVFPArgKind::Default)
     abiFloatType = EF_ARM_ABI_FLOAT_SOFT;
   else if (config->armVFPArgs == ARMVFPArgKind::VFP)
     abiFloatType = EF_ARM_ABI_FLOAT_HARD;
 
   // We don't currently use any features incompatible with EF_ARM_EABI_VER5,
   // but we don't have any firm guarantees of conformance. Linux AArch64
   // kernels (as of 2016) require an EABI version to be set.
   return EF_ARM_EABI_VER5 | abiFloatType;
 }
 
 RelExpr ARM::getRelExpr(RelType type, const Symbol &s,
                         const uint8_t *loc) const {
   switch (type) {
   case R_ARM_ABS32:
   case R_ARM_MOVW_ABS_NC:
   case R_ARM_MOVT_ABS:
   case R_ARM_THM_MOVW_ABS_NC:
   case R_ARM_THM_MOVT_ABS:
     return R_ABS;
   case R_ARM_THM_JUMP8:
   case R_ARM_THM_JUMP11:
     return R_PC;
   case R_ARM_CALL:
   case R_ARM_JUMP24:
   case R_ARM_PC24:
   case R_ARM_PLT32:
   case R_ARM_PREL31:
   case R_ARM_THM_JUMP19:
   case R_ARM_THM_JUMP24:
   case R_ARM_THM_CALL:
     return R_PLT_PC;
   case R_ARM_GOTOFF32:
     // (S + A) - GOT_ORG
     return R_GOTREL;
   case R_ARM_GOT_BREL:
     // GOT(S) + A - GOT_ORG
     return R_GOT_OFF;
   case R_ARM_GOT_PREL:
   case R_ARM_TLS_IE32:
     // GOT(S) + A - P
     return R_GOT_PC;
   case R_ARM_SBREL32:
     return R_ARM_SBREL;
   case R_ARM_TARGET1:
     return config->target1Rel ? R_PC : R_ABS;
   case R_ARM_TARGET2:
     if (config->target2 == Target2Policy::Rel)
       return R_PC;
     if (config->target2 == Target2Policy::Abs)
       return R_ABS;
     return R_GOT_PC;
   case R_ARM_TLS_GD32:
     return R_TLSGD_PC;
   case R_ARM_TLS_LDM32:
     return R_TLSLD_PC;
   case R_ARM_TLS_LDO32:
     return R_DTPREL;
   case R_ARM_BASE_PREL:
     // B(S) + A - P
     // FIXME: currently B(S) assumed to be .got, this may not hold for all
     // platforms.
     return R_GOTONLY_PC;
   case R_ARM_MOVW_PREL_NC:
   case R_ARM_MOVT_PREL:
   case R_ARM_REL32:
   case R_ARM_THM_MOVW_PREL_NC:
   case R_ARM_THM_MOVT_PREL:
     return R_PC;
   case R_ARM_ALU_PC_G0:
   case R_ARM_ALU_PC_G0_NC:
   case R_ARM_ALU_PC_G1:
   case R_ARM_ALU_PC_G1_NC:
   case R_ARM_ALU_PC_G2:
   case R_ARM_LDR_PC_G0:
   case R_ARM_LDR_PC_G1:
   case R_ARM_LDR_PC_G2:
   case R_ARM_LDRS_PC_G0:
   case R_ARM_LDRS_PC_G1:
   case R_ARM_LDRS_PC_G2:
   case R_ARM_THM_ALU_PREL_11_0:
   case R_ARM_THM_PC8:
   case R_ARM_THM_PC12:
     return R_ARM_PCA;
   case R_ARM_MOVW_BREL_NC:
   case R_ARM_MOVW_BREL:
   case R_ARM_MOVT_BREL:
   case R_ARM_THM_MOVW_BREL_NC:
   case R_ARM_THM_MOVW_BREL:
   case R_ARM_THM_MOVT_BREL:
     return R_ARM_SBREL;
   case R_ARM_NONE:
     return R_NONE;
   case R_ARM_TLS_LE32:
     return R_TPREL;
   case R_ARM_V4BX:
     // V4BX is just a marker to indicate there's a "bx rN" instruction at the
     // given address. It can be used to implement a special linker mode which
     // rewrites ARMv4T inputs to ARMv4. Since we support only ARMv4 input and
     // not ARMv4 output, we can just ignore it.
     return R_NONE;
   default:
     error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) +
           ") against symbol " + toString(s));
     return R_NONE;
   }
 }
 
 RelType ARM::getDynRel(RelType type) const {
   if ((type == R_ARM_ABS32) || (type == R_ARM_TARGET1 && !config->target1Rel))
     return R_ARM_ABS32;
   return R_ARM_NONE;
 }
 
 void ARM::writeGotPlt(uint8_t *buf, const Symbol &) const {
   write32le(buf, in.plt->getVA());
 }
 
 void ARM::writeIgotPlt(uint8_t *buf, const Symbol &s) const {
   // An ARM entry is the address of the ifunc resolver function.
   write32le(buf, s.getVA());
 }
 
 // Long form PLT Header that does not have any restrictions on the displacement
 // of the .plt from the .plt.got.
 static void writePltHeaderLong(uint8_t *buf) {
   const uint8_t pltData[] = {
       0x04, 0xe0, 0x2d, 0xe5, //     str lr, [sp,#-4]!
       0x04, 0xe0, 0x9f, 0xe5, //     ldr lr, L2
       0x0e, 0xe0, 0x8f, 0xe0, // L1: add lr, pc, lr
       0x08, 0xf0, 0xbe, 0xe5, //     ldr pc, [lr, #8]
       0x00, 0x00, 0x00, 0x00, // L2: .word   &(.got.plt) - L1 - 8
       0xd4, 0xd4, 0xd4, 0xd4, //     Pad to 32-byte boundary
       0xd4, 0xd4, 0xd4, 0xd4, //     Pad to 32-byte boundary
       0xd4, 0xd4, 0xd4, 0xd4};
   memcpy(buf, pltData, sizeof(pltData));
   uint64_t gotPlt = in.gotPlt->getVA();
   uint64_t l1 = in.plt->getVA() + 8;
   write32le(buf + 16, gotPlt - l1 - 8);
 }
 
 // The default PLT header requires the .plt.got to be within 128 Mb of the
 // .plt in the positive direction.
 void ARM::writePltHeader(uint8_t *buf) const {
   // Use a similar sequence to that in writePlt(), the difference is the calling
   // conventions mean we use lr instead of ip. The PLT entry is responsible for
   // saving lr on the stack, the dynamic loader is responsible for reloading
   // it.
   const uint32_t pltData[] = {
       0xe52de004, // L1: str lr, [sp,#-4]!
       0xe28fe600, //     add lr, pc,  #0x0NN00000 &(.got.plt - L1 - 4)
       0xe28eea00, //     add lr, lr,  #0x000NN000 &(.got.plt - L1 - 4)
       0xe5bef000, //     ldr pc, [lr, #0x00000NNN] &(.got.plt -L1 - 4)
   };
 
   uint64_t offset = in.gotPlt->getVA() - in.plt->getVA() - 4;
   if (!llvm::isUInt<27>(offset)) {
     // We cannot encode the Offset, use the long form.
     writePltHeaderLong(buf);
     return;
   }
   write32le(buf + 0, pltData[0]);
   write32le(buf + 4, pltData[1] | ((offset >> 20) & 0xff));
   write32le(buf + 8, pltData[2] | ((offset >> 12) & 0xff));
   write32le(buf + 12, pltData[3] | (offset & 0xfff));
   memcpy(buf + 16, trapInstr.data(), 4); // Pad to 32-byte boundary
   memcpy(buf + 20, trapInstr.data(), 4);
   memcpy(buf + 24, trapInstr.data(), 4);
   memcpy(buf + 28, trapInstr.data(), 4);
 }
 
 void ARM::addPltHeaderSymbols(InputSection &isec) const {
   addSyntheticLocal("$a", STT_NOTYPE, 0, 0, isec);
   addSyntheticLocal("$d", STT_NOTYPE, 16, 0, isec);
 }
 
 // Long form PLT entries that do not have any restrictions on the displacement
 // of the .plt from the .plt.got.
 static void writePltLong(uint8_t *buf, uint64_t gotPltEntryAddr,
                          uint64_t pltEntryAddr) {
   const uint8_t pltData[] = {
       0x04, 0xc0, 0x9f, 0xe5, //     ldr ip, L2
       0x0f, 0xc0, 0x8c, 0xe0, // L1: add ip, ip, pc
       0x00, 0xf0, 0x9c, 0xe5, //     ldr pc, [ip]
       0x00, 0x00, 0x00, 0x00, // L2: .word   Offset(&(.plt.got) - L1 - 8
   };
   memcpy(buf, pltData, sizeof(pltData));
   uint64_t l1 = pltEntryAddr + 4;
   write32le(buf + 12, gotPltEntryAddr - l1 - 8);
 }
 
 // The default PLT entries require the .plt.got to be within 128 Mb of the
 // .plt in the positive direction.
 void ARM::writePlt(uint8_t *buf, const Symbol &sym,
                    uint64_t pltEntryAddr) const {
   // The PLT entry is similar to the example given in Appendix A of ELF for
   // the Arm Architecture. Instead of using the Group Relocations to find the
   // optimal rotation for the 8-bit immediate used in the add instructions we
   // hard code the most compact rotations for simplicity. This saves a load
   // instruction over the long plt sequences.
   const uint32_t pltData[] = {
       0xe28fc600, // L1: add ip, pc,  #0x0NN00000  Offset(&(.plt.got) - L1 - 8
       0xe28cca00, //     add ip, ip,  #0x000NN000  Offset(&(.plt.got) - L1 - 8
       0xe5bcf000, //     ldr pc, [ip, #0x00000NNN] Offset(&(.plt.got) - L1 - 8
   };
 
   uint64_t offset = sym.getGotPltVA() - pltEntryAddr - 8;
   if (!llvm::isUInt<27>(offset)) {
     // We cannot encode the Offset, use the long form.
     writePltLong(buf, sym.getGotPltVA(), pltEntryAddr);
     return;
   }
   write32le(buf + 0, pltData[0] | ((offset >> 20) & 0xff));
   write32le(buf + 4, pltData[1] | ((offset >> 12) & 0xff));
   write32le(buf + 8, pltData[2] | (offset & 0xfff));
   memcpy(buf + 12, trapInstr.data(), 4); // Pad to 16-byte boundary
 }
 
 void ARM::addPltSymbols(InputSection &isec, uint64_t off) const {
   addSyntheticLocal("$a", STT_NOTYPE, off, 0, isec);
   addSyntheticLocal("$d", STT_NOTYPE, off + 12, 0, isec);
 }
 
 bool ARM::needsThunk(RelExpr expr, RelType type, const InputFile *file,
                      uint64_t branchAddr, const Symbol &s,
                      int64_t a) const {
-  // If S is an undefined weak symbol and does not have a PLT entry then it
-  // will be resolved as a branch to the next instruction.
-  if (s.isUndefWeak() && !s.isInPlt())
+  // If s is an undefined weak symbol and does not have a PLT entry then it will
+  // be resolved as a branch to the next instruction. If it is hidden, its
+  // binding has been converted to local, so we just check isUndefined() here. A
+  // undefined non-weak symbol will have been errored.
+  if (s.isUndefined() && !s.isInPlt())
     return false;
   // A state change from ARM to Thumb and vice versa must go through an
   // interworking thunk if the relocation type is not R_ARM_CALL or
   // R_ARM_THM_CALL.
   switch (type) {
   case R_ARM_PC24:
   case R_ARM_PLT32:
   case R_ARM_JUMP24:
     // Source is ARM, all PLT entries are ARM so no interworking required.
     // Otherwise we need to interwork if STT_FUNC Symbol has bit 0 set (Thumb).
     if (s.isFunc() && expr == R_PC && (s.getVA() & 1))
       return true;
     LLVM_FALLTHROUGH;
   case R_ARM_CALL: {
     uint64_t dst = (expr == R_PLT_PC) ? s.getPltVA() : s.getVA();
     return !inBranchRange(type, branchAddr, dst + a);
   }
   case R_ARM_THM_JUMP19:
   case R_ARM_THM_JUMP24:
     // Source is Thumb, all PLT entries are ARM so interworking is required.
     // Otherwise we need to interwork if STT_FUNC Symbol has bit 0 clear (ARM).
     if (expr == R_PLT_PC || (s.isFunc() && (s.getVA() & 1) == 0))
       return true;
     LLVM_FALLTHROUGH;
   case R_ARM_THM_CALL: {
     uint64_t dst = (expr == R_PLT_PC) ? s.getPltVA() : s.getVA();
     return !inBranchRange(type, branchAddr, dst + a);
   }
   }
   return false;
 }
 
 uint32_t ARM::getThunkSectionSpacing() const {
   // The placing of pre-created ThunkSections is controlled by the value
   // thunkSectionSpacing returned by getThunkSectionSpacing(). The aim is to
   // place the ThunkSection such that all branches from the InputSections
   // prior to the ThunkSection can reach a Thunk placed at the end of the
   // ThunkSection. Graphically:
   // | up to thunkSectionSpacing .text input sections |
   // | ThunkSection                                   |
   // | up to thunkSectionSpacing .text input sections |
   // | ThunkSection                                   |
 
   // Pre-created ThunkSections are spaced roughly 16MiB apart on ARMv7. This
   // is to match the most common expected case of a Thumb 2 encoded BL, BLX or
   // B.W:
   // ARM B, BL, BLX range +/- 32MiB
   // Thumb B.W, BL, BLX range +/- 16MiB
   // Thumb B<cc>.W range +/- 1MiB
   // If a branch cannot reach a pre-created ThunkSection a new one will be
   // created so we can handle the rare cases of a Thumb 2 conditional branch.
   // We intentionally use a lower size for thunkSectionSpacing than the maximum
   // branch range so the end of the ThunkSection is more likely to be within
   // range of the branch instruction that is furthest away. The value we shorten
   // thunkSectionSpacing by is set conservatively to allow us to create 16,384
   // 12 byte Thunks at any offset in a ThunkSection without risk of a branch to
   // one of the Thunks going out of range.
 
   // On Arm the thunkSectionSpacing depends on the range of the Thumb Branch
   // range. On earlier Architectures such as ARMv4, ARMv5 and ARMv6 (except
   // ARMv6T2) the range is +/- 4MiB.
 
   return (config->armJ1J2BranchEncoding) ? 0x1000000 - 0x30000
                                          : 0x400000 - 0x7500;
 }
 
 bool ARM::inBranchRange(RelType type, uint64_t src, uint64_t dst) const {
   if ((dst & 0x1) == 0)
     // Destination is ARM, if ARM caller then Src is already 4-byte aligned.
     // If Thumb Caller (BLX) the Src address has bottom 2 bits cleared to ensure
     // destination will be 4 byte aligned.
     src &= ~0x3;
   else
     // Bit 0 == 1 denotes Thumb state, it is not part of the range.
     dst &= ~0x1;
 
   int64_t offset = dst - src;
   switch (type) {
   case R_ARM_PC24:
   case R_ARM_PLT32:
   case R_ARM_JUMP24:
   case R_ARM_CALL:
     return llvm::isInt<26>(offset);
   case R_ARM_THM_JUMP19:
     return llvm::isInt<21>(offset);
   case R_ARM_THM_JUMP24:
   case R_ARM_THM_CALL:
     return config->armJ1J2BranchEncoding ? llvm::isInt<25>(offset)
                                          : llvm::isInt<23>(offset);
   default:
     return true;
   }
 }
 
 // Helper to produce message text when LLD detects that a CALL relocation to
 // a non STT_FUNC symbol that may result in incorrect interworking between ARM
 // or Thumb.
 static void stateChangeWarning(uint8_t *loc, RelType relt, const Symbol &s) {
   assert(!s.isFunc());
   const ErrorPlace place = getErrorPlace(loc);
   std::string hint;
   if (!place.srcLoc.empty())
     hint = "; " + place.srcLoc;
   if (s.isSection()) {
     // Section symbols must be defined and in a section. Users cannot change
     // the type. Use the section name as getName() returns an empty string.
     warn(place.loc + "branch and link relocation: " + toString(relt) +
          " to STT_SECTION symbol " + cast<Defined>(s).section->name +
          " ; interworking not performed" + hint);
   } else {
     // Warn with hint on how to alter the symbol type.
     warn(getErrorLocation(loc) + "branch and link relocation: " +
          toString(relt) + " to non STT_FUNC symbol: " + s.getName() +
          " interworking not performed; consider using directive '.type " +
          s.getName() +
          ", %function' to give symbol type STT_FUNC if interworking between "
          "ARM and Thumb is required" +
          hint);
   }
 }
 
 // Rotate a 32-bit unsigned value right by a specified amt of bits.
 static uint32_t rotr32(uint32_t val, uint32_t amt) {
   assert(amt < 32 && "Invalid rotate amount");
   return (val >> amt) | (val << ((32 - amt) & 31));
 }
 
 static std::pair<uint32_t, uint32_t> getRemAndLZForGroup(unsigned group,
                                                          uint32_t val) {
   uint32_t rem, lz;
   do {
     lz = llvm::countLeadingZeros(val) & ~1;
     rem = val;
     if (lz == 32) // implies rem == 0
       break;
     val &= 0xffffff >> lz;
   } while (group--);
   return {rem, lz};
 }
 
 static void encodeAluGroup(uint8_t *loc, const Relocation &rel, uint64_t val,
                            int group, bool check) {
   // ADD/SUB (immediate) add = bit23, sub = bit22
   // immediate field carries is a 12-bit modified immediate, made up of a 4-bit
   // even rotate right and an 8-bit immediate.
   uint32_t opcode = 0x00800000;
   if (val >> 63) {
     opcode = 0x00400000;
     val = -val;
   }
   uint32_t imm, lz;
   std::tie(imm, lz) = getRemAndLZForGroup(group, val);
   uint32_t rot = 0;
   if (lz < 24) {
     imm = rotr32(imm, 24 - lz);
     rot = (lz + 8) << 7;
   }
   if (check && imm > 0xff)
     error(getErrorLocation(loc) + "unencodeable immediate " + Twine(val).str() +
           " for relocation " + toString(rel.type));
   write32le(loc, (read32le(loc) & 0xff3ff000) | opcode | rot | (imm & 0xff));
 }
 
 static void encodeLdrGroup(uint8_t *loc, const Relocation &rel, uint64_t val,
                            int group) {
   // R_ARM_LDR_PC_Gn is S + A - P, we have ((S + A) | T) - P, if S is a
   // function then addr is 0 (modulo 2) and Pa is 0 (modulo 4) so we can clear
   // bottom bit to recover S + A - P.
   if (rel.sym->isFunc())
     val &= ~0x1;
   // LDR (literal) u = bit23
   uint32_t opcode = 0x00800000;
   if (val >> 63) {
     opcode = 0x0;
     val = -val;
   }
   uint32_t imm = getRemAndLZForGroup(group, val).first;
   checkUInt(loc, imm, 12, rel);
   write32le(loc, (read32le(loc) & 0xff7ff000) | opcode | imm);
 }
 
 static void encodeLdrsGroup(uint8_t *loc, const Relocation &rel, uint64_t val,
                             int group) {
   // R_ARM_LDRS_PC_Gn is S + A - P, we have ((S + A) | T) - P, if S is a
   // function then addr is 0 (modulo 2) and Pa is 0 (modulo 4) so we can clear
   // bottom bit to recover S + A - P.
   if (rel.sym->isFunc())
     val &= ~0x1;
   // LDRD/LDRH/LDRSB/LDRSH (literal) u = bit23
   uint32_t opcode = 0x00800000;
   if (val >> 63) {
     opcode = 0x0;
     val = -val;
   }
   uint32_t imm = getRemAndLZForGroup(group, val).first;
   checkUInt(loc, imm, 8, rel);
   write32le(loc, (read32le(loc) & 0xff7ff0f0) | opcode | ((imm & 0xf0) << 4) |
                      (imm & 0xf));
 }
 
 void ARM::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
   switch (rel.type) {
   case R_ARM_ABS32:
   case R_ARM_BASE_PREL:
   case R_ARM_GOTOFF32:
   case R_ARM_GOT_BREL:
   case R_ARM_GOT_PREL:
   case R_ARM_REL32:
   case R_ARM_RELATIVE:
   case R_ARM_SBREL32:
   case R_ARM_TARGET1:
   case R_ARM_TARGET2:
   case R_ARM_TLS_GD32:
   case R_ARM_TLS_IE32:
   case R_ARM_TLS_LDM32:
   case R_ARM_TLS_LDO32:
   case R_ARM_TLS_LE32:
   case R_ARM_TLS_TPOFF32:
   case R_ARM_TLS_DTPOFF32:
     write32le(loc, val);
     break;
   case R_ARM_PREL31:
     checkInt(loc, val, 31, rel);
     write32le(loc, (read32le(loc) & 0x80000000) | (val & ~0x80000000));
     break;
   case R_ARM_CALL: {
     // R_ARM_CALL is used for BL and BLX instructions, for symbols of type
     // STT_FUNC we choose whether to write a BL or BLX depending on the
     // value of bit 0 of Val. With bit 0 == 1 denoting Thumb. If the symbol is
     // not of type STT_FUNC then we must preserve the original instruction.
     // PLT entries are always ARM state so we know we don't need to interwork.
     assert(rel.sym); // R_ARM_CALL is always reached via relocate().
     bool bit0Thumb = val & 1;
     bool isBlx = (read32le(loc) & 0xfe000000) == 0xfa000000;
     // lld 10.0 and before always used bit0Thumb when deciding to write a BLX
     // even when type not STT_FUNC.
     if (!rel.sym->isFunc() && isBlx != bit0Thumb)
       stateChangeWarning(loc, rel.type, *rel.sym);
     if (rel.sym->isFunc() ? bit0Thumb : isBlx) {
       // The BLX encoding is 0xfa:H:imm24 where Val = imm24:H:'1'
       checkInt(loc, val, 26, rel);
       write32le(loc, 0xfa000000 |                    // opcode
                          ((val & 2) << 23) |         // H
                          ((val >> 2) & 0x00ffffff)); // imm24
       break;
     }
     // BLX (always unconditional) instruction to an ARM Target, select an
     // unconditional BL.
     write32le(loc, 0xeb000000 | (read32le(loc) & 0x00ffffff));
     // fall through as BL encoding is shared with B
   }
     LLVM_FALLTHROUGH;
   case R_ARM_JUMP24:
   case R_ARM_PC24:
   case R_ARM_PLT32:
     checkInt(loc, val, 26, rel);
     write32le(loc, (read32le(loc) & ~0x00ffffff) | ((val >> 2) & 0x00ffffff));
     break;
   case R_ARM_THM_JUMP8:
     // We do a 9 bit check because val is right-shifted by 1 bit.
     checkInt(loc, val, 9, rel);
     write16le(loc, (read32le(loc) & 0xff00) | ((val >> 1) & 0x00ff));
     break;
   case R_ARM_THM_JUMP11:
     // We do a 12 bit check because val is right-shifted by 1 bit.
     checkInt(loc, val, 12, rel);
     write16le(loc, (read32le(loc) & 0xf800) | ((val >> 1) & 0x07ff));
     break;
   case R_ARM_THM_JUMP19:
     // Encoding T3: Val = S:J2:J1:imm6:imm11:0
     checkInt(loc, val, 21, rel);
     write16le(loc,
               (read16le(loc) & 0xfbc0) |   // opcode cond
                   ((val >> 10) & 0x0400) | // S
                   ((val >> 12) & 0x003f)); // imm6
     write16le(loc + 2,
               0x8000 |                    // opcode
                   ((val >> 8) & 0x0800) | // J2
                   ((val >> 5) & 0x2000) | // J1
                   ((val >> 1) & 0x07ff)); // imm11
     break;
   case R_ARM_THM_CALL: {
     // R_ARM_THM_CALL is used for BL and BLX instructions, for symbols of type
     // STT_FUNC we choose whether to write a BL or BLX depending on the
     // value of bit 0 of Val. With bit 0 == 0 denoting ARM, if the symbol is
     // not of type STT_FUNC then we must preserve the original instruction.
     // PLT entries are always ARM state so we know we need to interwork.
     assert(rel.sym); // R_ARM_THM_CALL is always reached via relocate().
     bool bit0Thumb = val & 1;
     bool isBlx = (read16le(loc + 2) & 0x1000) == 0;
     // lld 10.0 and before always used bit0Thumb when deciding to write a BLX
     // even when type not STT_FUNC. PLT entries generated by LLD are always ARM.
     if (!rel.sym->isFunc() && !rel.sym->isInPlt() && isBlx == bit0Thumb)
       stateChangeWarning(loc, rel.type, *rel.sym);
     if (rel.sym->isFunc() || rel.sym->isInPlt() ? !bit0Thumb : isBlx) {
       // We are writing a BLX. Ensure BLX destination is 4-byte aligned. As
       // the BLX instruction may only be two byte aligned. This must be done
       // before overflow check.
       val = alignTo(val, 4);
       write16le(loc + 2, read16le(loc + 2) & ~0x1000);
     } else {
       write16le(loc + 2, (read16le(loc + 2) & ~0x1000) | 1 << 12);
     }
     if (!config->armJ1J2BranchEncoding) {
       // Older Arm architectures do not support R_ARM_THM_JUMP24 and have
       // different encoding rules and range due to J1 and J2 always being 1.
       checkInt(loc, val, 23, rel);
       write16le(loc,
                 0xf000 |                     // opcode
                     ((val >> 12) & 0x07ff)); // imm11
       write16le(loc + 2,
                 (read16le(loc + 2) & 0xd000) | // opcode
                     0x2800 |                   // J1 == J2 == 1
                     ((val >> 1) & 0x07ff));    // imm11
       break;
     }
   }
     // Fall through as rest of encoding is the same as B.W
     LLVM_FALLTHROUGH;
   case R_ARM_THM_JUMP24:
     // Encoding B  T4, BL T1, BLX T2: Val = S:I1:I2:imm10:imm11:0
     checkInt(loc, val, 25, rel);
     write16le(loc,
               0xf000 |                     // opcode
                   ((val >> 14) & 0x0400) | // S
                   ((val >> 12) & 0x03ff)); // imm10
     write16le(loc + 2,
               (read16le(loc + 2) & 0xd000) |                  // opcode
                   (((~(val >> 10)) ^ (val >> 11)) & 0x2000) | // J1
                   (((~(val >> 11)) ^ (val >> 13)) & 0x0800) | // J2
                   ((val >> 1) & 0x07ff));                     // imm11
     break;
   case R_ARM_MOVW_ABS_NC:
   case R_ARM_MOVW_PREL_NC:
   case R_ARM_MOVW_BREL_NC:
     write32le(loc, (read32le(loc) & ~0x000f0fff) | ((val & 0xf000) << 4) |
                        (val & 0x0fff));
     break;
   case R_ARM_MOVT_ABS:
   case R_ARM_MOVT_PREL:
   case R_ARM_MOVT_BREL:
     write32le(loc, (read32le(loc) & ~0x000f0fff) |
                        (((val >> 16) & 0xf000) << 4) | ((val >> 16) & 0xfff));
     break;
   case R_ARM_THM_MOVT_ABS:
   case R_ARM_THM_MOVT_PREL:
   case R_ARM_THM_MOVT_BREL:
     // Encoding T1: A = imm4:i:imm3:imm8
     write16le(loc,
               0xf2c0 |                     // opcode
                   ((val >> 17) & 0x0400) | // i
                   ((val >> 28) & 0x000f)); // imm4
     write16le(loc + 2,
               (read16le(loc + 2) & 0x8f00) | // opcode
                   ((val >> 12) & 0x7000) |   // imm3
                   ((val >> 16) & 0x00ff));   // imm8
     break;
   case R_ARM_THM_MOVW_ABS_NC:
   case R_ARM_THM_MOVW_PREL_NC:
   case R_ARM_THM_MOVW_BREL_NC:
     // Encoding T3: A = imm4:i:imm3:imm8
     write16le(loc,
               0xf240 |                     // opcode
                   ((val >> 1) & 0x0400) |  // i
                   ((val >> 12) & 0x000f)); // imm4
     write16le(loc + 2,
               (read16le(loc + 2) & 0x8f00) | // opcode
                   ((val << 4) & 0x7000) |    // imm3
                   (val & 0x00ff));           // imm8
     break;
   case R_ARM_ALU_PC_G0:
     encodeAluGroup(loc, rel, val, 0, true);
     break;
   case R_ARM_ALU_PC_G0_NC:
     encodeAluGroup(loc, rel, val, 0, false);
     break;
   case R_ARM_ALU_PC_G1:
     encodeAluGroup(loc, rel, val, 1, true);
     break;
   case R_ARM_ALU_PC_G1_NC:
     encodeAluGroup(loc, rel, val, 1, false);
     break;
   case R_ARM_ALU_PC_G2:
     encodeAluGroup(loc, rel, val, 2, true);
     break;
   case R_ARM_LDR_PC_G0:
     encodeLdrGroup(loc, rel, val, 0);
     break;
   case R_ARM_LDR_PC_G1:
     encodeLdrGroup(loc, rel, val, 1);
     break;
   case R_ARM_LDR_PC_G2:
     encodeLdrGroup(loc, rel, val, 2);
     break;
   case R_ARM_LDRS_PC_G0:
     encodeLdrsGroup(loc, rel, val, 0);
     break;
   case R_ARM_LDRS_PC_G1:
     encodeLdrsGroup(loc, rel, val, 1);
     break;
   case R_ARM_LDRS_PC_G2:
     encodeLdrsGroup(loc, rel, val, 2);
     break;
   case R_ARM_THM_ALU_PREL_11_0: {
     // ADR encoding T2 (sub), T3 (add) i:imm3:imm8
     int64_t imm = val;
     uint16_t sub = 0;
     if (imm < 0) {
       imm = -imm;
       sub = 0x00a0;
     }
     checkUInt(loc, imm, 12, rel);
     write16le(loc, (read16le(loc) & 0xfb0f) | sub | (imm & 0x800) >> 1);
     write16le(loc + 2,
               (read16le(loc + 2) & 0x8f00) | (imm & 0x700) << 4 | (imm & 0xff));
     break;
   }
   case R_ARM_THM_PC8:
     // ADR and LDR literal encoding T1 positive offset only imm8:00
     // R_ARM_THM_PC8 is S + A - Pa, we have ((S + A) | T) - Pa, if S is a
     // function then addr is 0 (modulo 2) and Pa is 0 (modulo 4) so we can clear
     // bottom bit to recover S + A - Pa.
     if (rel.sym->isFunc())
       val &= ~0x1;
     checkUInt(loc, val, 10, rel);
     checkAlignment(loc, val, 4, rel);
     write16le(loc, (read16le(loc) & 0xff00) | (val & 0x3fc) >> 2);
     break;
   case R_ARM_THM_PC12: {
     // LDR (literal) encoding T2, add = (U == '1') imm12
     // imm12 is unsigned
     // R_ARM_THM_PC12 is S + A - Pa, we have ((S + A) | T) - Pa, if S is a
     // function then addr is 0 (modulo 2) and Pa is 0 (modulo 4) so we can clear
     // bottom bit to recover S + A - Pa.
     if (rel.sym->isFunc())
       val &= ~0x1;
     int64_t imm12 = val;
     uint16_t u = 0x0080;
     if (imm12 < 0) {
       imm12 = -imm12;
       u = 0;
     }
     checkUInt(loc, imm12, 12, rel);
     write16le(loc, read16le(loc) | u);
     write16le(loc + 2, (read16le(loc + 2) & 0xf000) | imm12);
     break;
   }
   default:
     llvm_unreachable("unknown relocation");
   }
 }
 
 int64_t ARM::getImplicitAddend(const uint8_t *buf, RelType type) const {
   switch (type) {
   default:
     internalLinkerError(getErrorLocation(buf),
                         "cannot read addend for relocation " + toString(type));
     return 0;
   case R_ARM_ABS32:
   case R_ARM_BASE_PREL:
   case R_ARM_GLOB_DAT:
   case R_ARM_GOTOFF32:
   case R_ARM_GOT_BREL:
   case R_ARM_GOT_PREL:
   case R_ARM_IRELATIVE:
   case R_ARM_REL32:
   case R_ARM_RELATIVE:
   case R_ARM_SBREL32:
   case R_ARM_TARGET1:
   case R_ARM_TARGET2:
   case R_ARM_TLS_DTPMOD32:
   case R_ARM_TLS_DTPOFF32:
   case R_ARM_TLS_GD32:
   case R_ARM_TLS_IE32:
   case R_ARM_TLS_LDM32:
   case R_ARM_TLS_LE32:
   case R_ARM_TLS_LDO32:
   case R_ARM_TLS_TPOFF32:
     return SignExtend64<32>(read32le(buf));
   case R_ARM_PREL31:
     return SignExtend64<31>(read32le(buf));
   case R_ARM_CALL:
   case R_ARM_JUMP24:
   case R_ARM_PC24:
   case R_ARM_PLT32:
     return SignExtend64<26>(read32le(buf) << 2);
   case R_ARM_THM_JUMP8:
     return SignExtend64<9>(read16le(buf) << 1);
   case R_ARM_THM_JUMP11:
     return SignExtend64<12>(read16le(buf) << 1);
   case R_ARM_THM_JUMP19: {
     // Encoding T3: A = S:J2:J1:imm10:imm6:0
     uint16_t hi = read16le(buf);
     uint16_t lo = read16le(buf + 2);
     return SignExtend64<20>(((hi & 0x0400) << 10) | // S
                             ((lo & 0x0800) << 8) |  // J2
                             ((lo & 0x2000) << 5) |  // J1
                             ((hi & 0x003f) << 12) | // imm6
                             ((lo & 0x07ff) << 1));  // imm11:0
   }
   case R_ARM_THM_CALL:
     if (!config->armJ1J2BranchEncoding) {
       // Older Arm architectures do not support R_ARM_THM_JUMP24 and have
       // different encoding rules and range due to J1 and J2 always being 1.
       uint16_t hi = read16le(buf);
       uint16_t lo = read16le(buf + 2);
       return SignExtend64<22>(((hi & 0x7ff) << 12) | // imm11
                               ((lo & 0x7ff) << 1));  // imm11:0
       break;
     }
     LLVM_FALLTHROUGH;
   case R_ARM_THM_JUMP24: {
     // Encoding B T4, BL T1, BLX T2: A = S:I1:I2:imm10:imm11:0
     // I1 = NOT(J1 EOR S), I2 = NOT(J2 EOR S)
     uint16_t hi = read16le(buf);
     uint16_t lo = read16le(buf + 2);
     return SignExtend64<24>(((hi & 0x0400) << 14) |                    // S
                             (~((lo ^ (hi << 3)) << 10) & 0x00800000) | // I1
                             (~((lo ^ (hi << 1)) << 11) & 0x00400000) | // I2
                             ((hi & 0x003ff) << 12) |                   // imm0
                             ((lo & 0x007ff) << 1)); // imm11:0
   }
   // ELF for the ARM Architecture 4.6.1.1 the implicit addend for MOVW and
   // MOVT is in the range -32768 <= A < 32768
   case R_ARM_MOVW_ABS_NC:
   case R_ARM_MOVT_ABS:
   case R_ARM_MOVW_PREL_NC:
   case R_ARM_MOVT_PREL:
   case R_ARM_MOVW_BREL_NC:
   case R_ARM_MOVT_BREL: {
     uint64_t val = read32le(buf) & 0x000f0fff;
     return SignExtend64<16>(((val & 0x000f0000) >> 4) | (val & 0x00fff));
   }
   case R_ARM_THM_MOVW_ABS_NC:
   case R_ARM_THM_MOVT_ABS:
   case R_ARM_THM_MOVW_PREL_NC:
   case R_ARM_THM_MOVT_PREL:
   case R_ARM_THM_MOVW_BREL_NC:
   case R_ARM_THM_MOVT_BREL: {
     // Encoding T3: A = imm4:i:imm3:imm8
     uint16_t hi = read16le(buf);
     uint16_t lo = read16le(buf + 2);
     return SignExtend64<16>(((hi & 0x000f) << 12) | // imm4
                             ((hi & 0x0400) << 1) |  // i
                             ((lo & 0x7000) >> 4) |  // imm3
                             (lo & 0x00ff));         // imm8
   }
   case R_ARM_ALU_PC_G0:
   case R_ARM_ALU_PC_G0_NC:
   case R_ARM_ALU_PC_G1:
   case R_ARM_ALU_PC_G1_NC:
   case R_ARM_ALU_PC_G2: {
     // 12-bit immediate is a modified immediate made up of a 4-bit even
     // right rotation and 8-bit constant. After the rotation the value
     // is zero-extended. When bit 23 is set the instruction is an add, when
     // bit 22 is set it is a sub.
     uint32_t instr = read32le(buf);
     uint32_t val = rotr32(instr & 0xff, ((instr & 0xf00) >> 8) * 2);
     return (instr & 0x00400000) ? -val : val;
   }
   case R_ARM_LDR_PC_G0:
   case R_ARM_LDR_PC_G1:
   case R_ARM_LDR_PC_G2: {
     // ADR (literal) add = bit23, sub = bit22
     // LDR (literal) u = bit23 unsigned imm12
     bool u = read32le(buf) & 0x00800000;
     uint32_t imm12 = read32le(buf) & 0xfff;
     return u ? imm12 : -imm12;
   }
   case R_ARM_LDRS_PC_G0:
   case R_ARM_LDRS_PC_G1:
   case R_ARM_LDRS_PC_G2: {
     // LDRD/LDRH/LDRSB/LDRSH (literal) u = bit23 unsigned imm8
     uint32_t opcode = read32le(buf);
     bool u = opcode & 0x00800000;
     uint32_t imm4l = opcode & 0xf;
     uint32_t imm4h = (opcode & 0xf00) >> 4;
     return u ? (imm4h | imm4l) : -(imm4h | imm4l);
   }
   case R_ARM_THM_ALU_PREL_11_0: {
     // Thumb2 ADR, which is an alias for a sub or add instruction with an
     // unsigned immediate.
     // ADR encoding T2 (sub), T3 (add) i:imm3:imm8
     uint16_t hi = read16le(buf);
     uint16_t lo = read16le(buf + 2);
     uint64_t imm = (hi & 0x0400) << 1 | // i
                    (lo & 0x7000) >> 4 | // imm3
                    (lo & 0x00ff);       // imm8
     // For sub, addend is negative, add is positive.
     return (hi & 0x00f0) ? -imm : imm;
   }
   case R_ARM_THM_PC8:
     // ADR and LDR (literal) encoding T1
     // From ELF for the ARM Architecture the initial signed addend is formed
     // from an unsigned field using expression (((imm8:00 + 4) & 0x3ff) – 4)
     // this trick permits the PC bias of -4 to be encoded using imm8 = 0xff
     return ((((read16le(buf) & 0xff) << 2) + 4) & 0x3ff) - 4;
   case R_ARM_THM_PC12: {
     // LDR (literal) encoding T2, add = (U == '1') imm12
     bool u = read16le(buf) & 0x0080;
     uint64_t imm12 = read16le(buf + 2) & 0x0fff;
     return u ? imm12 : -imm12;
   }
   case R_ARM_NONE:
   case R_ARM_V4BX:
   case R_ARM_JUMP_SLOT:
     // These relocations are defined as not having an implicit addend.
     return 0;
   }
 }
 
 TargetInfo *elf::getARMTargetInfo() {
   static ARM target;
   return &target;
 }
diff --git a/contrib/llvm-project/lld/ELF/InputSection.cpp b/contrib/llvm-project/lld/ELF/InputSection.cpp
index 4b047f75ad69..490254c995c8 100644
--- a/contrib/llvm-project/lld/ELF/InputSection.cpp
+++ b/contrib/llvm-project/lld/ELF/InputSection.cpp
@@ -1,1492 +1,1494 @@
 //===- InputSection.cpp ---------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "InputSection.h"
 #include "Config.h"
 #include "EhFrame.h"
 #include "InputFiles.h"
 #include "LinkerScript.h"
 #include "OutputSections.h"
 #include "Relocations.h"
 #include "SymbolTable.h"
 #include "Symbols.h"
 #include "SyntheticSections.h"
 #include "Target.h"
 #include "Thunks.h"
 #include "lld/Common/CommonLinkerContext.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Threading.h"
 #include "llvm/Support/xxhash.h"
 #include <algorithm>
 #include <mutex>
 #include <set>
 #include <vector>
 
 using namespace llvm;
 using namespace llvm::ELF;
 using namespace llvm::object;
 using namespace llvm::support;
 using namespace llvm::support::endian;
 using namespace llvm::sys;
 using namespace lld;
 using namespace lld::elf;
 
 SmallVector<InputSectionBase *, 0> elf::inputSections;
 DenseSet<std::pair<const Symbol *, uint64_t>> elf::ppc64noTocRelax;
 
 // Returns a string to construct an error message.
 std::string lld::toString(const InputSectionBase *sec) {
   return (toString(sec->file) + ":(" + sec->name + ")").str();
 }
 
 template <class ELFT>
 static ArrayRef<uint8_t> getSectionContents(ObjFile<ELFT> &file,
                                             const typename ELFT::Shdr &hdr) {
   if (hdr.sh_type == SHT_NOBITS)
     return makeArrayRef<uint8_t>(nullptr, hdr.sh_size);
   return check(file.getObj().getSectionContents(hdr));
 }
 
 InputSectionBase::InputSectionBase(InputFile *file, uint64_t flags,
                                    uint32_t type, uint64_t entsize,
                                    uint32_t link, uint32_t info,
                                    uint32_t alignment, ArrayRef<uint8_t> data,
                                    StringRef name, Kind sectionKind)
     : SectionBase(sectionKind, name, flags, entsize, alignment, type, info,
                   link),
       file(file), rawData(data) {
   // In order to reduce memory allocation, we assume that mergeable
   // sections are smaller than 4 GiB, which is not an unreasonable
   // assumption as of 2017.
   if (sectionKind == SectionBase::Merge && rawData.size() > UINT32_MAX)
     error(toString(this) + ": section too large");
 
   // The ELF spec states that a value of 0 means the section has
   // no alignment constraints.
   uint32_t v = std::max<uint32_t>(alignment, 1);
   if (!isPowerOf2_64(v))
     fatal(toString(this) + ": sh_addralign is not a power of 2");
   this->alignment = v;
 
   // In ELF, each section can be compressed by zlib, and if compressed,
   // section name may be mangled by appending "z" (e.g. ".zdebug_info").
   // If that's the case, demangle section name so that we can handle a
   // section as if it weren't compressed.
   if ((flags & SHF_COMPRESSED) || name.startswith(".zdebug")) {
     if (!zlib::isAvailable())
       error(toString(file) + ": contains a compressed section, " +
             "but zlib is not available");
     invokeELFT(parseCompressedHeader);
   }
 }
 
 // Drop SHF_GROUP bit unless we are producing a re-linkable object file.
 // SHF_GROUP is a marker that a section belongs to some comdat group.
 // That flag doesn't make sense in an executable.
 static uint64_t getFlags(uint64_t flags) {
   flags &= ~(uint64_t)SHF_INFO_LINK;
   if (!config->relocatable)
     flags &= ~(uint64_t)SHF_GROUP;
   return flags;
 }
 
 template <class ELFT>
 InputSectionBase::InputSectionBase(ObjFile<ELFT> &file,
                                    const typename ELFT::Shdr &hdr,
                                    StringRef name, Kind sectionKind)
     : InputSectionBase(&file, getFlags(hdr.sh_flags), hdr.sh_type,
                        hdr.sh_entsize, hdr.sh_link, hdr.sh_info,
                        hdr.sh_addralign, getSectionContents(file, hdr), name,
                        sectionKind) {
   // We reject object files having insanely large alignments even though
   // they are allowed by the spec. I think 4GB is a reasonable limitation.
   // We might want to relax this in the future.
   if (hdr.sh_addralign > UINT32_MAX)
     fatal(toString(&file) + ": section sh_addralign is too large");
 }
 
 size_t InputSectionBase::getSize() const {
   if (auto *s = dyn_cast<SyntheticSection>(this))
     return s->getSize();
   if (uncompressedSize >= 0)
     return uncompressedSize;
   return rawData.size() - bytesDropped;
 }
 
 void InputSectionBase::uncompress() const {
   size_t size = uncompressedSize;
   char *uncompressedBuf;
   {
     static std::mutex mu;
     std::lock_guard<std::mutex> lock(mu);
     uncompressedBuf = bAlloc().Allocate<char>(size);
   }
 
   if (Error e = zlib::uncompress(toStringRef(rawData), uncompressedBuf, size))
     fatal(toString(this) +
           ": uncompress failed: " + llvm::toString(std::move(e)));
   rawData = makeArrayRef((uint8_t *)uncompressedBuf, size);
   uncompressedSize = -1;
 }
 
 template <class ELFT> RelsOrRelas<ELFT> InputSectionBase::relsOrRelas() const {
   if (relSecIdx == 0)
     return {};
   RelsOrRelas<ELFT> ret;
   typename ELFT::Shdr shdr =
       cast<ELFFileBase>(file)->getELFShdrs<ELFT>()[relSecIdx];
   if (shdr.sh_type == SHT_REL) {
     ret.rels = makeArrayRef(reinterpret_cast<const typename ELFT::Rel *>(
                                 file->mb.getBufferStart() + shdr.sh_offset),
                             shdr.sh_size / sizeof(typename ELFT::Rel));
   } else {
     assert(shdr.sh_type == SHT_RELA);
     ret.relas = makeArrayRef(reinterpret_cast<const typename ELFT::Rela *>(
                                  file->mb.getBufferStart() + shdr.sh_offset),
                              shdr.sh_size / sizeof(typename ELFT::Rela));
   }
   return ret;
 }
 
 uint64_t SectionBase::getOffset(uint64_t offset) const {
   switch (kind()) {
   case Output: {
     auto *os = cast<OutputSection>(this);
     // For output sections we treat offset -1 as the end of the section.
     return offset == uint64_t(-1) ? os->size : offset;
   }
   case Regular:
   case Synthetic:
     return cast<InputSection>(this)->outSecOff + offset;
   case EHFrame:
     // The file crtbeginT.o has relocations pointing to the start of an empty
     // .eh_frame that is known to be the first in the link. It does that to
     // identify the start of the output .eh_frame.
     return offset;
   case Merge:
     const MergeInputSection *ms = cast<MergeInputSection>(this);
     if (InputSection *isec = ms->getParent())
       return isec->outSecOff + ms->getParentOffset(offset);
     return ms->getParentOffset(offset);
   }
   llvm_unreachable("invalid section kind");
 }
 
 uint64_t SectionBase::getVA(uint64_t offset) const {
   const OutputSection *out = getOutputSection();
   return (out ? out->addr : 0) + getOffset(offset);
 }
 
 OutputSection *SectionBase::getOutputSection() {
   InputSection *sec;
   if (auto *isec = dyn_cast<InputSection>(this))
     sec = isec;
   else if (auto *ms = dyn_cast<MergeInputSection>(this))
     sec = ms->getParent();
   else if (auto *eh = dyn_cast<EhInputSection>(this))
     sec = eh->getParent();
   else
     return cast<OutputSection>(this);
   return sec ? sec->getParent() : nullptr;
 }
 
 // When a section is compressed, `rawData` consists with a header followed
 // by zlib-compressed data. This function parses a header to initialize
 // `uncompressedSize` member and remove the header from `rawData`.
 template <typename ELFT> void InputSectionBase::parseCompressedHeader() {
   // Old-style header
   if (!(flags & SHF_COMPRESSED)) {
     assert(name.startswith(".zdebug"));
     if (!toStringRef(rawData).startswith("ZLIB")) {
       error(toString(this) + ": corrupted compressed section header");
       return;
     }
     rawData = rawData.slice(4);
 
     if (rawData.size() < 8) {
       error(toString(this) + ": corrupted compressed section header");
       return;
     }
 
     uncompressedSize = read64be(rawData.data());
     rawData = rawData.slice(8);
 
     // Restore the original section name.
     // (e.g. ".zdebug_info" -> ".debug_info")
     name = saver().save("." + name.substr(2));
     return;
   }
 
   flags &= ~(uint64_t)SHF_COMPRESSED;
 
   // New-style header
   if (rawData.size() < sizeof(typename ELFT::Chdr)) {
     error(toString(this) + ": corrupted compressed section");
     return;
   }
 
   auto *hdr = reinterpret_cast<const typename ELFT::Chdr *>(rawData.data());
   if (hdr->ch_type != ELFCOMPRESS_ZLIB) {
     error(toString(this) + ": unsupported compression type");
     return;
   }
 
   uncompressedSize = hdr->ch_size;
   alignment = std::max<uint32_t>(hdr->ch_addralign, 1);
   rawData = rawData.slice(sizeof(*hdr));
 }
 
 InputSection *InputSectionBase::getLinkOrderDep() const {
   assert(flags & SHF_LINK_ORDER);
   if (!link)
     return nullptr;
   return cast<InputSection>(file->getSections()[link]);
 }
 
 // Find a function symbol that encloses a given location.
 Defined *InputSectionBase::getEnclosingFunction(uint64_t offset) {
   for (Symbol *b : file->getSymbols())
     if (Defined *d = dyn_cast<Defined>(b))
       if (d->section == this && d->type == STT_FUNC && d->value <= offset &&
           offset < d->value + d->size)
         return d;
   return nullptr;
 }
 
 // Returns an object file location string. Used to construct an error message.
 std::string InputSectionBase::getLocation(uint64_t offset) {
   std::string secAndOffset =
       (name + "+0x" + Twine::utohexstr(offset) + ")").str();
 
   // We don't have file for synthetic sections.
   if (file == nullptr)
     return (config->outputFile + ":(" + secAndOffset).str();
 
   std::string filename = toString(file);
   if (Defined *d = getEnclosingFunction(offset))
     return filename + ":(function " + toString(*d) + ": " + secAndOffset;
 
   return filename + ":(" + secAndOffset;
 }
 
 // This function is intended to be used for constructing an error message.
 // The returned message looks like this:
 //
 //   foo.c:42 (/home/alice/possibly/very/long/path/foo.c:42)
 //
 //  Returns an empty string if there's no way to get line info.
 std::string InputSectionBase::getSrcMsg(const Symbol &sym, uint64_t offset) {
   return file->getSrcMsg(sym, *this, offset);
 }
 
 // Returns a filename string along with an optional section name. This
 // function is intended to be used for constructing an error
 // message. The returned message looks like this:
 //
 //   path/to/foo.o:(function bar)
 //
 // or
 //
 //   path/to/foo.o:(function bar) in archive path/to/bar.a
 std::string InputSectionBase::getObjMsg(uint64_t off) {
   std::string filename = std::string(file->getName());
 
   std::string archive;
   if (!file->archiveName.empty())
     archive = (" in archive " + file->archiveName).str();
 
   // Find a symbol that encloses a given location.
   for (Symbol *b : file->getSymbols())
     if (auto *d = dyn_cast<Defined>(b))
       if (d->section == this && d->value <= off && off < d->value + d->size)
         return filename + ":(" + toString(*d) + ")" + archive;
 
   // If there's no symbol, print out the offset in the section.
   return (filename + ":(" + name + "+0x" + utohexstr(off) + ")" + archive)
       .str();
 }
 
 InputSection InputSection::discarded(nullptr, 0, 0, 0, ArrayRef<uint8_t>(), "");
 
 InputSection::InputSection(InputFile *f, uint64_t flags, uint32_t type,
                            uint32_t alignment, ArrayRef<uint8_t> data,
                            StringRef name, Kind k)
     : InputSectionBase(f, flags, type,
                        /*Entsize*/ 0, /*Link*/ 0, /*Info*/ 0, alignment, data,
                        name, k) {}
 
 template <class ELFT>
 InputSection::InputSection(ObjFile<ELFT> &f, const typename ELFT::Shdr &header,
                            StringRef name)
     : InputSectionBase(f, header, name, InputSectionBase::Regular) {}
 
 bool InputSection::classof(const SectionBase *s) {
   return s->kind() == SectionBase::Regular ||
          s->kind() == SectionBase::Synthetic;
 }
 
 OutputSection *InputSection::getParent() const {
   return cast_or_null<OutputSection>(parent);
 }
 
 // Copy SHT_GROUP section contents. Used only for the -r option.
 template <class ELFT> void InputSection::copyShtGroup(uint8_t *buf) {
   // ELFT::Word is the 32-bit integral type in the target endianness.
   using u32 = typename ELFT::Word;
   ArrayRef<u32> from = getDataAs<u32>();
   auto *to = reinterpret_cast<u32 *>(buf);
 
   // The first entry is not a section number but a flag.
   *to++ = from[0];
 
   // Adjust section numbers because section numbers in an input object files are
   // different in the output. We also need to handle combined or discarded
   // members.
   ArrayRef<InputSectionBase *> sections = file->getSections();
   DenseSet<uint32_t> seen;
   for (uint32_t idx : from.slice(1)) {
     OutputSection *osec = sections[idx]->getOutputSection();
     if (osec && seen.insert(osec->sectionIndex).second)
       *to++ = osec->sectionIndex;
   }
 }
 
 InputSectionBase *InputSection::getRelocatedSection() const {
   if (!file || (type != SHT_RELA && type != SHT_REL))
     return nullptr;
   ArrayRef<InputSectionBase *> sections = file->getSections();
   return sections[info];
 }
 
 // This is used for -r and --emit-relocs. We can't use memcpy to copy
 // relocations because we need to update symbol table offset and section index
 // for each relocation. So we copy relocations one by one.
 template <class ELFT, class RelTy>
 void InputSection::copyRelocations(uint8_t *buf, ArrayRef<RelTy> rels) {
   const TargetInfo &target = *elf::target;
   InputSectionBase *sec = getRelocatedSection();
 
   for (const RelTy &rel : rels) {
     RelType type = rel.getType(config->isMips64EL);
     const ObjFile<ELFT> *file = getFile<ELFT>();
     Symbol &sym = file->getRelocTargetSym(rel);
 
     auto *p = reinterpret_cast<typename ELFT::Rela *>(buf);
     buf += sizeof(RelTy);
 
     if (RelTy::IsRela)
       p->r_addend = getAddend<ELFT>(rel);
 
     // Output section VA is zero for -r, so r_offset is an offset within the
     // section, but for --emit-relocs it is a virtual address.
     p->r_offset = sec->getVA(rel.r_offset);
     p->setSymbolAndType(in.symTab->getSymbolIndex(&sym), type,
                         config->isMips64EL);
 
     if (sym.type == STT_SECTION) {
       // We combine multiple section symbols into only one per
       // section. This means we have to update the addend. That is
       // trivial for Elf_Rela, but for Elf_Rel we have to write to the
       // section data. We do that by adding to the Relocation vector.
 
       // .eh_frame is horribly special and can reference discarded sections. To
       // avoid having to parse and recreate .eh_frame, we just replace any
       // relocation in it pointing to discarded sections with R_*_NONE, which
       // hopefully creates a frame that is ignored at runtime. Also, don't warn
       // on .gcc_except_table and debug sections.
       //
       // See the comment in maybeReportUndefined for PPC32 .got2 and PPC64 .toc
       auto *d = dyn_cast<Defined>(&sym);
       if (!d) {
         if (!isDebugSection(*sec) && sec->name != ".eh_frame" &&
             sec->name != ".gcc_except_table" && sec->name != ".got2" &&
             sec->name != ".toc") {
           uint32_t secIdx = cast<Undefined>(sym).discardedSecIdx;
           Elf_Shdr_Impl<ELFT> sec = file->template getELFShdrs<ELFT>()[secIdx];
           warn("relocation refers to a discarded section: " +
                CHECK(file->getObj().getSectionName(sec), file) +
                "\n>>> referenced by " + getObjMsg(p->r_offset));
         }
         p->setSymbolAndType(0, 0, false);
         continue;
       }
       SectionBase *section = d->section;
       if (!section->isLive()) {
         p->setSymbolAndType(0, 0, false);
         continue;
       }
 
       int64_t addend = getAddend<ELFT>(rel);
       const uint8_t *bufLoc = sec->data().begin() + rel.r_offset;
       if (!RelTy::IsRela)
         addend = target.getImplicitAddend(bufLoc, type);
 
       if (config->emachine == EM_MIPS &&
           target.getRelExpr(type, sym, bufLoc) == R_MIPS_GOTREL) {
         // Some MIPS relocations depend on "gp" value. By default,
         // this value has 0x7ff0 offset from a .got section. But
         // relocatable files produced by a compiler or a linker
         // might redefine this default value and we must use it
         // for a calculation of the relocation result. When we
         // generate EXE or DSO it's trivial. Generating a relocatable
         // output is more difficult case because the linker does
         // not calculate relocations in this mode and loses
         // individual "gp" values used by each input object file.
         // As a workaround we add the "gp" value to the relocation
         // addend and save it back to the file.
         addend += sec->getFile<ELFT>()->mipsGp0;
       }
 
       if (RelTy::IsRela)
         p->r_addend = sym.getVA(addend) - section->getOutputSection()->addr;
       else if (config->relocatable && type != target.noneRel)
         sec->relocations.push_back({R_ABS, type, rel.r_offset, addend, &sym});
     } else if (config->emachine == EM_PPC && type == R_PPC_PLTREL24 &&
                p->r_addend >= 0x8000 && sec->file->ppc32Got2) {
       // Similar to R_MIPS_GPREL{16,32}. If the addend of R_PPC_PLTREL24
       // indicates that r30 is relative to the input section .got2
       // (r_addend>=0x8000), after linking, r30 should be relative to the output
       // section .got2 . To compensate for the shift, adjust r_addend by
       // ppc32Got->outSecOff.
       p->r_addend += sec->file->ppc32Got2->outSecOff;
     }
   }
 }
 
 // The ARM and AArch64 ABI handle pc-relative relocations to undefined weak
 // references specially. The general rule is that the value of the symbol in
 // this context is the address of the place P. A further special case is that
 // branch relocations to an undefined weak reference resolve to the next
 // instruction.
 static uint32_t getARMUndefinedRelativeWeakVA(RelType type, uint32_t a,
                                               uint32_t p) {
   switch (type) {
   // Unresolved branch relocations to weak references resolve to next
   // instruction, this will be either 2 or 4 bytes on from P.
   case R_ARM_THM_JUMP8:
   case R_ARM_THM_JUMP11:
     return p + 2 + a;
   case R_ARM_CALL:
   case R_ARM_JUMP24:
   case R_ARM_PC24:
   case R_ARM_PLT32:
   case R_ARM_PREL31:
   case R_ARM_THM_JUMP19:
   case R_ARM_THM_JUMP24:
     return p + 4 + a;
   case R_ARM_THM_CALL:
     // We don't want an interworking BLX to ARM
     return p + 5 + a;
   // Unresolved non branch pc-relative relocations
   // R_ARM_TARGET2 which can be resolved relatively is not present as it never
   // targets a weak-reference.
   case R_ARM_MOVW_PREL_NC:
   case R_ARM_MOVT_PREL:
   case R_ARM_REL32:
   case R_ARM_THM_ALU_PREL_11_0:
   case R_ARM_THM_MOVW_PREL_NC:
   case R_ARM_THM_MOVT_PREL:
   case R_ARM_THM_PC12:
     return p + a;
   // p + a is unrepresentable as negative immediates can't be encoded.
   case R_ARM_THM_PC8:
     return p;
   }
   llvm_unreachable("ARM pc-relative relocation expected\n");
 }
 
 // The comment above getARMUndefinedRelativeWeakVA applies to this function.
 static uint64_t getAArch64UndefinedRelativeWeakVA(uint64_t type, uint64_t p) {
   switch (type) {
   // Unresolved branch relocations to weak references resolve to next
   // instruction, this is 4 bytes on from P.
   case R_AARCH64_CALL26:
   case R_AARCH64_CONDBR19:
   case R_AARCH64_JUMP26:
   case R_AARCH64_TSTBR14:
     return p + 4;
   // Unresolved non branch pc-relative relocations
   case R_AARCH64_PREL16:
   case R_AARCH64_PREL32:
   case R_AARCH64_PREL64:
   case R_AARCH64_ADR_PREL_LO21:
   case R_AARCH64_LD_PREL_LO19:
   case R_AARCH64_PLT32:
     return p;
   }
   llvm_unreachable("AArch64 pc-relative relocation expected\n");
 }
 
 static uint64_t getRISCVUndefinedRelativeWeakVA(uint64_t type, uint64_t p) {
   switch (type) {
   case R_RISCV_BRANCH:
   case R_RISCV_JAL:
   case R_RISCV_CALL:
   case R_RISCV_CALL_PLT:
   case R_RISCV_RVC_BRANCH:
   case R_RISCV_RVC_JUMP:
     return p;
   default:
     return 0;
   }
 }
 
 // ARM SBREL relocations are of the form S + A - B where B is the static base
 // The ARM ABI defines base to be "addressing origin of the output segment
 // defining the symbol S". We defined the "addressing origin"/static base to be
 // the base of the PT_LOAD segment containing the Sym.
 // The procedure call standard only defines a Read Write Position Independent
 // RWPI variant so in practice we should expect the static base to be the base
 // of the RW segment.
 static uint64_t getARMStaticBase(const Symbol &sym) {
   OutputSection *os = sym.getOutputSection();
   if (!os || !os->ptLoad || !os->ptLoad->firstSec)
     fatal("SBREL relocation to " + sym.getName() + " without static base");
   return os->ptLoad->firstSec->addr;
 }
 
 // For R_RISCV_PC_INDIRECT (R_RISCV_PCREL_LO12_{I,S}), the symbol actually
 // points the corresponding R_RISCV_PCREL_HI20 relocation, and the target VA
 // is calculated using PCREL_HI20's symbol.
 //
 // This function returns the R_RISCV_PCREL_HI20 relocation from
 // R_RISCV_PCREL_LO12's symbol and addend.
 static Relocation *getRISCVPCRelHi20(const Symbol *sym, uint64_t addend) {
   const Defined *d = cast<Defined>(sym);
   if (!d->section) {
     error("R_RISCV_PCREL_LO12 relocation points to an absolute symbol: " +
           sym->getName());
     return nullptr;
   }
   InputSection *isec = cast<InputSection>(d->section);
 
   if (addend != 0)
     warn("non-zero addend in R_RISCV_PCREL_LO12 relocation to " +
          isec->getObjMsg(d->value) + " is ignored");
 
   // Relocations are sorted by offset, so we can use std::equal_range to do
   // binary search.
   Relocation r;
   r.offset = d->value;
   auto range =
       std::equal_range(isec->relocations.begin(), isec->relocations.end(), r,
                        [](const Relocation &lhs, const Relocation &rhs) {
                          return lhs.offset < rhs.offset;
                        });
 
   for (auto it = range.first; it != range.second; ++it)
     if (it->type == R_RISCV_PCREL_HI20 || it->type == R_RISCV_GOT_HI20 ||
         it->type == R_RISCV_TLS_GD_HI20 || it->type == R_RISCV_TLS_GOT_HI20)
       return &*it;
 
   error("R_RISCV_PCREL_LO12 relocation points to " + isec->getObjMsg(d->value) +
         " without an associated R_RISCV_PCREL_HI20 relocation");
   return nullptr;
 }
 
 // A TLS symbol's virtual address is relative to the TLS segment. Add a
 // target-specific adjustment to produce a thread-pointer-relative offset.
 static int64_t getTlsTpOffset(const Symbol &s) {
   // On targets that support TLSDESC, _TLS_MODULE_BASE_@tpoff = 0.
   if (&s == ElfSym::tlsModuleBase)
     return 0;
 
   // There are 2 TLS layouts. Among targets we support, x86 uses TLS Variant 2
   // while most others use Variant 1. At run time TP will be aligned to p_align.
 
   // Variant 1. TP will be followed by an optional gap (which is the size of 2
   // pointers on ARM/AArch64, 0 on other targets), followed by alignment
   // padding, then the static TLS blocks. The alignment padding is added so that
   // (TP + gap + padding) is congruent to p_vaddr modulo p_align.
   //
   // Variant 2. Static TLS blocks, followed by alignment padding are placed
   // before TP. The alignment padding is added so that (TP - padding -
   // p_memsz) is congruent to p_vaddr modulo p_align.
   PhdrEntry *tls = Out::tlsPhdr;
   switch (config->emachine) {
     // Variant 1.
   case EM_ARM:
   case EM_AARCH64:
     return s.getVA(0) + config->wordsize * 2 +
            ((tls->p_vaddr - config->wordsize * 2) & (tls->p_align - 1));
   case EM_MIPS:
   case EM_PPC:
   case EM_PPC64:
     // Adjusted Variant 1. TP is placed with a displacement of 0x7000, which is
     // to allow a signed 16-bit offset to reach 0x1000 of TCB/thread-library
     // data and 0xf000 of the program's TLS segment.
     return s.getVA(0) + (tls->p_vaddr & (tls->p_align - 1)) - 0x7000;
   case EM_RISCV:
     return s.getVA(0) + (tls->p_vaddr & (tls->p_align - 1));
 
     // Variant 2.
   case EM_HEXAGON:
   case EM_SPARCV9:
   case EM_386:
   case EM_X86_64:
     return s.getVA(0) - tls->p_memsz -
            ((-tls->p_vaddr - tls->p_memsz) & (tls->p_align - 1));
   default:
     llvm_unreachable("unhandled Config->EMachine");
   }
 }
 
 uint64_t InputSectionBase::getRelocTargetVA(const InputFile *file, RelType type,
                                             int64_t a, uint64_t p,
                                             const Symbol &sym, RelExpr expr) {
   switch (expr) {
   case R_ABS:
   case R_DTPREL:
   case R_RELAX_TLS_LD_TO_LE_ABS:
   case R_RELAX_GOT_PC_NOPIC:
   case R_RISCV_ADD:
     return sym.getVA(a);
   case R_ADDEND:
     return a;
   case R_ARM_SBREL:
     return sym.getVA(a) - getARMStaticBase(sym);
   case R_GOT:
   case R_RELAX_TLS_GD_TO_IE_ABS:
     return sym.getGotVA() + a;
   case R_GOTONLY_PC:
     return in.got->getVA() + a - p;
   case R_GOTPLTONLY_PC:
     return in.gotPlt->getVA() + a - p;
   case R_GOTREL:
   case R_PPC64_RELAX_TOC:
     return sym.getVA(a) - in.got->getVA();
   case R_GOTPLTREL:
     return sym.getVA(a) - in.gotPlt->getVA();
   case R_GOTPLT:
   case R_RELAX_TLS_GD_TO_IE_GOTPLT:
     return sym.getGotVA() + a - in.gotPlt->getVA();
   case R_TLSLD_GOT_OFF:
   case R_GOT_OFF:
   case R_RELAX_TLS_GD_TO_IE_GOT_OFF:
     return sym.getGotOffset() + a;
   case R_AARCH64_GOT_PAGE_PC:
   case R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC:
     return getAArch64Page(sym.getGotVA() + a) - getAArch64Page(p);
   case R_AARCH64_GOT_PAGE:
     return sym.getGotVA() + a - getAArch64Page(in.got->getVA());
   case R_GOT_PC:
   case R_RELAX_TLS_GD_TO_IE:
     return sym.getGotVA() + a - p;
   case R_MIPS_GOTREL:
     return sym.getVA(a) - in.mipsGot->getGp(file);
   case R_MIPS_GOT_GP:
     return in.mipsGot->getGp(file) + a;
   case R_MIPS_GOT_GP_PC: {
     // R_MIPS_LO16 expression has R_MIPS_GOT_GP_PC type iif the target
     // is _gp_disp symbol. In that case we should use the following
     // formula for calculation "AHL + GP - P + 4". For details see p. 4-19 at
     // ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf
     // microMIPS variants of these relocations use slightly different
     // expressions: AHL + GP - P + 3 for %lo() and AHL + GP - P - 1 for %hi()
     // to correctly handle less-significant bit of the microMIPS symbol.
     uint64_t v = in.mipsGot->getGp(file) + a - p;
     if (type == R_MIPS_LO16 || type == R_MICROMIPS_LO16)
       v += 4;
     if (type == R_MICROMIPS_LO16 || type == R_MICROMIPS_HI16)
       v -= 1;
     return v;
   }
   case R_MIPS_GOT_LOCAL_PAGE:
     // If relocation against MIPS local symbol requires GOT entry, this entry
     // should be initialized by 'page address'. This address is high 16-bits
     // of sum the symbol's value and the addend.
     return in.mipsGot->getVA() + in.mipsGot->getPageEntryOffset(file, sym, a) -
            in.mipsGot->getGp(file);
   case R_MIPS_GOT_OFF:
   case R_MIPS_GOT_OFF32:
     // In case of MIPS if a GOT relocation has non-zero addend this addend
     // should be applied to the GOT entry content not to the GOT entry offset.
     // That is why we use separate expression type.
     return in.mipsGot->getVA() + in.mipsGot->getSymEntryOffset(file, sym, a) -
            in.mipsGot->getGp(file);
   case R_MIPS_TLSGD:
     return in.mipsGot->getVA() + in.mipsGot->getGlobalDynOffset(file, sym) -
            in.mipsGot->getGp(file);
   case R_MIPS_TLSLD:
     return in.mipsGot->getVA() + in.mipsGot->getTlsIndexOffset(file) -
            in.mipsGot->getGp(file);
   case R_AARCH64_PAGE_PC: {
     uint64_t val = sym.isUndefWeak() ? p + a : sym.getVA(a);
     return getAArch64Page(val) - getAArch64Page(p);
   }
   case R_RISCV_PC_INDIRECT: {
     if (const Relocation *hiRel = getRISCVPCRelHi20(&sym, a))
       return getRelocTargetVA(file, hiRel->type, hiRel->addend, sym.getVA(),
                               *hiRel->sym, hiRel->expr);
     return 0;
   }
   case R_PC:
   case R_ARM_PCA: {
     uint64_t dest;
     if (expr == R_ARM_PCA)
       // Some PC relative ARM (Thumb) relocations align down the place.
       p = p & 0xfffffffc;
-    if (sym.isUndefWeak()) {
+    if (sym.isUndefined()) {
       // On ARM and AArch64 a branch to an undefined weak resolves to the next
       // instruction, otherwise the place. On RISCV, resolve an undefined weak
       // to the same instruction to cause an infinite loop (making the user
       // aware of the issue) while ensuring no overflow.
+      // Note: if the symbol is hidden, its binding has been converted to local,
+      // so we just check isUndefined() here.
       if (config->emachine == EM_ARM)
         dest = getARMUndefinedRelativeWeakVA(type, a, p);
       else if (config->emachine == EM_AARCH64)
         dest = getAArch64UndefinedRelativeWeakVA(type, p) + a;
       else if (config->emachine == EM_PPC)
         dest = p;
       else if (config->emachine == EM_RISCV)
         dest = getRISCVUndefinedRelativeWeakVA(type, p) + a;
       else
         dest = sym.getVA(a);
     } else {
       dest = sym.getVA(a);
     }
     return dest - p;
   }
   case R_PLT:
     return sym.getPltVA() + a;
   case R_PLT_PC:
   case R_PPC64_CALL_PLT:
     return sym.getPltVA() + a - p;
   case R_PLT_GOTPLT:
     return sym.getPltVA() + a - in.gotPlt->getVA();
   case R_PPC32_PLTREL:
     // R_PPC_PLTREL24 uses the addend (usually 0 or 0x8000) to indicate r30
     // stores _GLOBAL_OFFSET_TABLE_ or .got2+0x8000. The addend is ignored for
     // target VA computation.
     return sym.getPltVA() - p;
   case R_PPC64_CALL: {
     uint64_t symVA = sym.getVA(a);
     // If we have an undefined weak symbol, we might get here with a symbol
     // address of zero. That could overflow, but the code must be unreachable,
     // so don't bother doing anything at all.
     if (!symVA)
       return 0;
 
     // PPC64 V2 ABI describes two entry points to a function. The global entry
     // point is used for calls where the caller and callee (may) have different
     // TOC base pointers and r2 needs to be modified to hold the TOC base for
     // the callee. For local calls the caller and callee share the same
     // TOC base and so the TOC pointer initialization code should be skipped by
     // branching to the local entry point.
     return symVA - p + getPPC64GlobalEntryToLocalEntryOffset(sym.stOther);
   }
   case R_PPC64_TOCBASE:
     return getPPC64TocBase() + a;
   case R_RELAX_GOT_PC:
   case R_PPC64_RELAX_GOT_PC:
     return sym.getVA(a) - p;
   case R_RELAX_TLS_GD_TO_LE:
   case R_RELAX_TLS_IE_TO_LE:
   case R_RELAX_TLS_LD_TO_LE:
   case R_TPREL:
     // It is not very clear what to return if the symbol is undefined. With
     // --noinhibit-exec, even a non-weak undefined reference may reach here.
     // Just return A, which matches R_ABS, and the behavior of some dynamic
     // loaders.
     if (sym.isUndefined())
       return a;
     return getTlsTpOffset(sym) + a;
   case R_RELAX_TLS_GD_TO_LE_NEG:
   case R_TPREL_NEG:
     if (sym.isUndefined())
       return a;
     return -getTlsTpOffset(sym) + a;
   case R_SIZE:
     return sym.getSize() + a;
   case R_TLSDESC:
     return in.got->getTlsDescAddr(sym) + a;
   case R_TLSDESC_PC:
     return in.got->getTlsDescAddr(sym) + a - p;
   case R_TLSDESC_GOTPLT:
     return in.got->getTlsDescAddr(sym) + a - in.gotPlt->getVA();
   case R_AARCH64_TLSDESC_PAGE:
     return getAArch64Page(in.got->getTlsDescAddr(sym) + a) - getAArch64Page(p);
   case R_TLSGD_GOT:
     return in.got->getGlobalDynOffset(sym) + a;
   case R_TLSGD_GOTPLT:
     return in.got->getGlobalDynAddr(sym) + a - in.gotPlt->getVA();
   case R_TLSGD_PC:
     return in.got->getGlobalDynAddr(sym) + a - p;
   case R_TLSLD_GOTPLT:
     return in.got->getVA() + in.got->getTlsIndexOff() + a - in.gotPlt->getVA();
   case R_TLSLD_GOT:
     return in.got->getTlsIndexOff() + a;
   case R_TLSLD_PC:
     return in.got->getTlsIndexVA() + a - p;
   default:
     llvm_unreachable("invalid expression");
   }
 }
 
 // This function applies relocations to sections without SHF_ALLOC bit.
 // Such sections are never mapped to memory at runtime. Debug sections are
 // an example. Relocations in non-alloc sections are much easier to
 // handle than in allocated sections because it will never need complex
 // treatment such as GOT or PLT (because at runtime no one refers them).
 // So, we handle relocations for non-alloc sections directly in this
 // function as a performance optimization.
 template <class ELFT, class RelTy>
 void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef<RelTy> rels) {
   const unsigned bits = sizeof(typename ELFT::uint) * 8;
   const TargetInfo &target = *elf::target;
   const bool isDebug = isDebugSection(*this);
   const bool isDebugLocOrRanges =
       isDebug && (name == ".debug_loc" || name == ".debug_ranges");
   const bool isDebugLine = isDebug && name == ".debug_line";
   Optional<uint64_t> tombstone;
   for (const auto &patAndValue : llvm::reverse(config->deadRelocInNonAlloc))
     if (patAndValue.first.match(this->name)) {
       tombstone = patAndValue.second;
       break;
     }
 
   for (const RelTy &rel : rels) {
     RelType type = rel.getType(config->isMips64EL);
 
     // GCC 8.0 or earlier have a bug that they emit R_386_GOTPC relocations
     // against _GLOBAL_OFFSET_TABLE_ for .debug_info. The bug has been fixed
     // in 2017 (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82630), but we
     // need to keep this bug-compatible code for a while.
     if (config->emachine == EM_386 && type == R_386_GOTPC)
       continue;
 
     uint64_t offset = rel.r_offset;
     uint8_t *bufLoc = buf + offset;
     int64_t addend = getAddend<ELFT>(rel);
     if (!RelTy::IsRela)
       addend += target.getImplicitAddend(bufLoc, type);
 
     Symbol &sym = getFile<ELFT>()->getRelocTargetSym(rel);
     RelExpr expr = target.getRelExpr(type, sym, bufLoc);
     if (expr == R_NONE)
       continue;
 
     if (tombstone ||
         (isDebug && (type == target.symbolicRel || expr == R_DTPREL))) {
       // Resolve relocations in .debug_* referencing (discarded symbols or ICF
       // folded section symbols) to a tombstone value. Resolving to addend is
       // unsatisfactory because the result address range may collide with a
       // valid range of low address, or leave multiple CUs claiming ownership of
       // the same range of code, which may confuse consumers.
       //
       // To address the problems, we use -1 as a tombstone value for most
       // .debug_* sections. We have to ignore the addend because we don't want
       // to resolve an address attribute (which may have a non-zero addend) to
       // -1+addend (wrap around to a low address).
       //
       // R_DTPREL type relocations represent an offset into the dynamic thread
       // vector. The computed value is st_value plus a non-negative offset.
       // Negative values are invalid, so -1 can be used as the tombstone value.
       //
       // If the referenced symbol is discarded (made Undefined), or the
       // section defining the referenced symbol is garbage collected,
       // sym.getOutputSection() is nullptr. `ds->folded` catches the ICF folded
       // case. However, resolving a relocation in .debug_line to -1 would stop
       // debugger users from setting breakpoints on the folded-in function, so
       // exclude .debug_line.
       //
       // For pre-DWARF-v5 .debug_loc and .debug_ranges, -1 is a reserved value
       // (base address selection entry), use 1 (which is used by GNU ld for
       // .debug_ranges).
       //
       // TODO To reduce disruption, we use 0 instead of -1 as the tombstone
       // value. Enable -1 in a future release.
       auto *ds = dyn_cast<Defined>(&sym);
       if (!sym.getOutputSection() || (ds && ds->folded && !isDebugLine)) {
         // If -z dead-reloc-in-nonalloc= is specified, respect it.
         const uint64_t value = tombstone ? SignExtend64<bits>(*tombstone)
                                          : (isDebugLocOrRanges ? 1 : 0);
         target.relocateNoSym(bufLoc, type, value);
         continue;
       }
     }
 
     // For a relocatable link, only tombstone values are applied.
     if (config->relocatable)
       continue;
 
     if (expr == R_SIZE) {
       target.relocateNoSym(bufLoc, type,
                            SignExtend64<bits>(sym.getSize() + addend));
       continue;
     }
 
     // R_ABS/R_DTPREL and some other relocations can be used from non-SHF_ALLOC
     // sections.
     if (expr == R_ABS || expr == R_DTPREL || expr == R_GOTPLTREL ||
         expr == R_RISCV_ADD) {
       target.relocateNoSym(bufLoc, type, SignExtend64<bits>(sym.getVA(addend)));
       continue;
     }
 
     std::string msg = getLocation(offset) + ": has non-ABS relocation " +
                       toString(type) + " against symbol '" + toString(sym) +
                       "'";
     if (expr != R_PC && expr != R_ARM_PCA) {
       error(msg);
       return;
     }
 
     // If the control reaches here, we found a PC-relative relocation in a
     // non-ALLOC section. Since non-ALLOC section is not loaded into memory
     // at runtime, the notion of PC-relative doesn't make sense here. So,
     // this is a usage error. However, GNU linkers historically accept such
     // relocations without any errors and relocate them as if they were at
     // address 0. For bug-compatibilty, we accept them with warnings. We
     // know Steel Bank Common Lisp as of 2018 have this bug.
     warn(msg);
     target.relocateNoSym(
         bufLoc, type,
         SignExtend64<bits>(sym.getVA(addend - offset - outSecOff)));
   }
 }
 
 // This is used when '-r' is given.
 // For REL targets, InputSection::copyRelocations() may store artificial
 // relocations aimed to update addends. They are handled in relocateAlloc()
 // for allocatable sections, and this function does the same for
 // non-allocatable sections, such as sections with debug information.
 static void relocateNonAllocForRelocatable(InputSection *sec, uint8_t *buf) {
   const unsigned bits = config->is64 ? 64 : 32;
 
   for (const Relocation &rel : sec->relocations) {
     // InputSection::copyRelocations() adds only R_ABS relocations.
     assert(rel.expr == R_ABS);
     uint8_t *bufLoc = buf + rel.offset;
     uint64_t targetVA = SignExtend64(rel.sym->getVA(rel.addend), bits);
     target->relocate(bufLoc, rel, targetVA);
   }
 }
 
 template <class ELFT>
 void InputSectionBase::relocate(uint8_t *buf, uint8_t *bufEnd) {
   if ((flags & SHF_EXECINSTR) && LLVM_UNLIKELY(getFile<ELFT>()->splitStack))
     adjustSplitStackFunctionPrologues<ELFT>(buf, bufEnd);
 
   if (flags & SHF_ALLOC) {
     relocateAlloc(buf, bufEnd);
     return;
   }
 
   auto *sec = cast<InputSection>(this);
   if (config->relocatable)
     relocateNonAllocForRelocatable(sec, buf);
   // For a relocatable link, also call relocateNonAlloc() to rewrite applicable
   // locations with tombstone values.
   const RelsOrRelas<ELFT> rels = sec->template relsOrRelas<ELFT>();
   if (rels.areRelocsRel())
     sec->relocateNonAlloc<ELFT>(buf, rels.rels);
   else
     sec->relocateNonAlloc<ELFT>(buf, rels.relas);
 }
 
 void InputSectionBase::relocateAlloc(uint8_t *buf, uint8_t *bufEnd) {
   assert(flags & SHF_ALLOC);
   const unsigned bits = config->wordsize * 8;
   const TargetInfo &target = *elf::target;
   uint64_t lastPPCRelaxedRelocOff = UINT64_C(-1);
   AArch64Relaxer aarch64relaxer(relocations);
   for (size_t i = 0, size = relocations.size(); i != size; ++i) {
     const Relocation &rel = relocations[i];
     if (rel.expr == R_NONE)
       continue;
     uint64_t offset = rel.offset;
     uint8_t *bufLoc = buf + offset;
 
     uint64_t secAddr = getOutputSection()->addr;
     if (auto *sec = dyn_cast<InputSection>(this))
       secAddr += sec->outSecOff;
     const uint64_t addrLoc = secAddr + offset;
     const uint64_t targetVA =
         SignExtend64(getRelocTargetVA(file, rel.type, rel.addend, addrLoc,
                                       *rel.sym, rel.expr),
                      bits);
     switch (rel.expr) {
     case R_RELAX_GOT_PC:
     case R_RELAX_GOT_PC_NOPIC:
       target.relaxGot(bufLoc, rel, targetVA);
       break;
     case R_AARCH64_GOT_PAGE_PC:
       if (i + 1 < size && aarch64relaxer.tryRelaxAdrpLdr(
                               rel, relocations[i + 1], secAddr, buf)) {
         ++i;
         continue;
       }
       target.relocate(bufLoc, rel, targetVA);
       break;
     case R_AARCH64_PAGE_PC:
       if (i + 1 < size && aarch64relaxer.tryRelaxAdrpAdd(
                               rel, relocations[i + 1], secAddr, buf)) {
         ++i;
         continue;
       }
       target.relocate(bufLoc, rel, targetVA);
       break;
     case R_PPC64_RELAX_GOT_PC: {
       // The R_PPC64_PCREL_OPT relocation must appear immediately after
       // R_PPC64_GOT_PCREL34 in the relocations table at the same offset.
       // We can only relax R_PPC64_PCREL_OPT if we have also relaxed
       // the associated R_PPC64_GOT_PCREL34 since only the latter has an
       // associated symbol. So save the offset when relaxing R_PPC64_GOT_PCREL34
       // and only relax the other if the saved offset matches.
       if (rel.type == R_PPC64_GOT_PCREL34)
         lastPPCRelaxedRelocOff = offset;
       if (rel.type == R_PPC64_PCREL_OPT && offset != lastPPCRelaxedRelocOff)
         break;
       target.relaxGot(bufLoc, rel, targetVA);
       break;
     }
     case R_PPC64_RELAX_TOC:
       // rel.sym refers to the STT_SECTION symbol associated to the .toc input
       // section. If an R_PPC64_TOC16_LO (.toc + addend) references the TOC
       // entry, there may be R_PPC64_TOC16_HA not paired with
       // R_PPC64_TOC16_LO_DS. Don't relax. This loses some relaxation
       // opportunities but is safe.
       if (ppc64noTocRelax.count({rel.sym, rel.addend}) ||
           !tryRelaxPPC64TocIndirection(rel, bufLoc))
         target.relocate(bufLoc, rel, targetVA);
       break;
     case R_RELAX_TLS_IE_TO_LE:
       target.relaxTlsIeToLe(bufLoc, rel, targetVA);
       break;
     case R_RELAX_TLS_LD_TO_LE:
     case R_RELAX_TLS_LD_TO_LE_ABS:
       target.relaxTlsLdToLe(bufLoc, rel, targetVA);
       break;
     case R_RELAX_TLS_GD_TO_LE:
     case R_RELAX_TLS_GD_TO_LE_NEG:
       target.relaxTlsGdToLe(bufLoc, rel, targetVA);
       break;
     case R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC:
     case R_RELAX_TLS_GD_TO_IE:
     case R_RELAX_TLS_GD_TO_IE_ABS:
     case R_RELAX_TLS_GD_TO_IE_GOT_OFF:
     case R_RELAX_TLS_GD_TO_IE_GOTPLT:
       target.relaxTlsGdToIe(bufLoc, rel, targetVA);
       break;
     case R_PPC64_CALL:
       // If this is a call to __tls_get_addr, it may be part of a TLS
       // sequence that has been relaxed and turned into a nop. In this
       // case, we don't want to handle it as a call.
       if (read32(bufLoc) == 0x60000000) // nop
         break;
 
       // Patch a nop (0x60000000) to a ld.
       if (rel.sym->needsTocRestore) {
         // gcc/gfortran 5.4, 6.3 and earlier versions do not add nop for
         // recursive calls even if the function is preemptible. This is not
         // wrong in the common case where the function is not preempted at
         // runtime. Just ignore.
         if ((bufLoc + 8 > bufEnd || read32(bufLoc + 4) != 0x60000000) &&
             rel.sym->file != file) {
           // Use substr(6) to remove the "__plt_" prefix.
           errorOrWarn(getErrorLocation(bufLoc) + "call to " +
                       lld::toString(*rel.sym).substr(6) +
                       " lacks nop, can't restore toc");
           break;
         }
         write32(bufLoc + 4, 0xe8410018); // ld %r2, 24(%r1)
       }
       target.relocate(bufLoc, rel, targetVA);
       break;
     default:
       target.relocate(bufLoc, rel, targetVA);
       break;
     }
   }
 
   // Apply jumpInstrMods.  jumpInstrMods are created when the opcode of
   // a jmp insn must be modified to shrink the jmp insn or to flip the jmp
   // insn.  This is primarily used to relax and optimize jumps created with
   // basic block sections.
   if (jumpInstrMod) {
     target.applyJumpInstrMod(buf + jumpInstrMod->offset, jumpInstrMod->original,
                              jumpInstrMod->size);
   }
 }
 
 // For each function-defining prologue, find any calls to __morestack,
 // and replace them with calls to __morestack_non_split.
 static void switchMorestackCallsToMorestackNonSplit(
     DenseSet<Defined *> &prologues,
     SmallVector<Relocation *, 0> &morestackCalls) {
 
   // If the target adjusted a function's prologue, all calls to
   // __morestack inside that function should be switched to
   // __morestack_non_split.
   Symbol *moreStackNonSplit = symtab->find("__morestack_non_split");
   if (!moreStackNonSplit) {
     error("mixing split-stack objects requires a definition of "
           "__morestack_non_split");
     return;
   }
 
   // Sort both collections to compare addresses efficiently.
   llvm::sort(morestackCalls, [](const Relocation *l, const Relocation *r) {
     return l->offset < r->offset;
   });
   std::vector<Defined *> functions(prologues.begin(), prologues.end());
   llvm::sort(functions, [](const Defined *l, const Defined *r) {
     return l->value < r->value;
   });
 
   auto it = morestackCalls.begin();
   for (Defined *f : functions) {
     // Find the first call to __morestack within the function.
     while (it != morestackCalls.end() && (*it)->offset < f->value)
       ++it;
     // Adjust all calls inside the function.
     while (it != morestackCalls.end() && (*it)->offset < f->value + f->size) {
       (*it)->sym = moreStackNonSplit;
       ++it;
     }
   }
 }
 
 static bool enclosingPrologueAttempted(uint64_t offset,
                                        const DenseSet<Defined *> &prologues) {
   for (Defined *f : prologues)
     if (f->value <= offset && offset < f->value + f->size)
       return true;
   return false;
 }
 
 // If a function compiled for split stack calls a function not
 // compiled for split stack, then the caller needs its prologue
 // adjusted to ensure that the called function will have enough stack
 // available. Find those functions, and adjust their prologues.
 template <class ELFT>
 void InputSectionBase::adjustSplitStackFunctionPrologues(uint8_t *buf,
                                                          uint8_t *end) {
   DenseSet<Defined *> prologues;
   SmallVector<Relocation *, 0> morestackCalls;
 
   for (Relocation &rel : relocations) {
     // Ignore calls into the split-stack api.
     if (rel.sym->getName().startswith("__morestack")) {
       if (rel.sym->getName().equals("__morestack"))
         morestackCalls.push_back(&rel);
       continue;
     }
 
     // A relocation to non-function isn't relevant. Sometimes
     // __morestack is not marked as a function, so this check comes
     // after the name check.
     if (rel.sym->type != STT_FUNC)
       continue;
 
     // If the callee's-file was compiled with split stack, nothing to do.  In
     // this context, a "Defined" symbol is one "defined by the binary currently
     // being produced". So an "undefined" symbol might be provided by a shared
     // library. It is not possible to tell how such symbols were compiled, so be
     // conservative.
     if (Defined *d = dyn_cast<Defined>(rel.sym))
       if (InputSection *isec = cast_or_null<InputSection>(d->section))
         if (!isec || !isec->getFile<ELFT>() || isec->getFile<ELFT>()->splitStack)
           continue;
 
     if (enclosingPrologueAttempted(rel.offset, prologues))
       continue;
 
     if (Defined *f = getEnclosingFunction(rel.offset)) {
       prologues.insert(f);
       if (target->adjustPrologueForCrossSplitStack(buf + f->value, end,
                                                    f->stOther))
         continue;
       if (!getFile<ELFT>()->someNoSplitStack)
         error(lld::toString(this) + ": " + f->getName() +
               " (with -fsplit-stack) calls " + rel.sym->getName() +
               " (without -fsplit-stack), but couldn't adjust its prologue");
     }
   }
 
   if (target->needsMoreStackNonSplit)
     switchMorestackCallsToMorestackNonSplit(prologues, morestackCalls);
 }
 
 template <class ELFT> void InputSection::writeTo(uint8_t *buf) {
   if (auto *s = dyn_cast<SyntheticSection>(this)) {
     s->writeTo(buf);
     return;
   }
 
   if (LLVM_UNLIKELY(type == SHT_NOBITS))
     return;
   // If -r or --emit-relocs is given, then an InputSection
   // may be a relocation section.
   if (LLVM_UNLIKELY(type == SHT_RELA)) {
     copyRelocations<ELFT>(buf, getDataAs<typename ELFT::Rela>());
     return;
   }
   if (LLVM_UNLIKELY(type == SHT_REL)) {
     copyRelocations<ELFT>(buf, getDataAs<typename ELFT::Rel>());
     return;
   }
 
   // If -r is given, we may have a SHT_GROUP section.
   if (LLVM_UNLIKELY(type == SHT_GROUP)) {
     copyShtGroup<ELFT>(buf);
     return;
   }
 
   // If this is a compressed section, uncompress section contents directly
   // to the buffer.
   if (uncompressedSize >= 0) {
     size_t size = uncompressedSize;
     if (Error e = zlib::uncompress(toStringRef(rawData), (char *)buf, size))
       fatal(toString(this) +
             ": uncompress failed: " + llvm::toString(std::move(e)));
     uint8_t *bufEnd = buf + size;
     relocate<ELFT>(buf, bufEnd);
     return;
   }
 
   // Copy section contents from source object file to output file
   // and then apply relocations.
   memcpy(buf, rawData.data(), rawData.size());
   relocate<ELFT>(buf, buf + rawData.size());
 }
 
 void InputSection::replace(InputSection *other) {
   alignment = std::max(alignment, other->alignment);
 
   // When a section is replaced with another section that was allocated to
   // another partition, the replacement section (and its associated sections)
   // need to be placed in the main partition so that both partitions will be
   // able to access it.
   if (partition != other->partition) {
     partition = 1;
     for (InputSection *isec : dependentSections)
       isec->partition = 1;
   }
 
   other->repl = repl;
   other->markDead();
 }
 
 template <class ELFT>
 EhInputSection::EhInputSection(ObjFile<ELFT> &f,
                                const typename ELFT::Shdr &header,
                                StringRef name)
     : InputSectionBase(f, header, name, InputSectionBase::EHFrame) {}
 
 SyntheticSection *EhInputSection::getParent() const {
   return cast_or_null<SyntheticSection>(parent);
 }
 
 // Returns the index of the first relocation that points to a region between
 // Begin and Begin+Size.
 template <class IntTy, class RelTy>
 static unsigned getReloc(IntTy begin, IntTy size, const ArrayRef<RelTy> &rels,
                          unsigned &relocI) {
   // Start search from RelocI for fast access. That works because the
   // relocations are sorted in .eh_frame.
   for (unsigned n = rels.size(); relocI < n; ++relocI) {
     const RelTy &rel = rels[relocI];
     if (rel.r_offset < begin)
       continue;
 
     if (rel.r_offset < begin + size)
       return relocI;
     return -1;
   }
   return -1;
 }
 
 // .eh_frame is a sequence of CIE or FDE records.
 // This function splits an input section into records and returns them.
 template <class ELFT> void EhInputSection::split() {
   const RelsOrRelas<ELFT> rels = relsOrRelas<ELFT>();
   // getReloc expects the relocations to be sorted by r_offset. See the comment
   // in scanRelocs.
   if (rels.areRelocsRel()) {
     SmallVector<typename ELFT::Rel, 0> storage;
     split<ELFT>(sortRels(rels.rels, storage));
   } else {
     SmallVector<typename ELFT::Rela, 0> storage;
     split<ELFT>(sortRels(rels.relas, storage));
   }
 }
 
 template <class ELFT, class RelTy>
 void EhInputSection::split(ArrayRef<RelTy> rels) {
   ArrayRef<uint8_t> d = rawData;
   const char *msg = nullptr;
   unsigned relI = 0;
   while (!d.empty()) {
     if (d.size() < 4) {
       msg = "CIE/FDE too small";
       break;
     }
     uint64_t size = endian::read32<ELFT::TargetEndianness>(d.data());
     // If it is 0xFFFFFFFF, the next 8 bytes contain the size instead,
     // but we do not support that format yet.
     if (size == UINT32_MAX) {
       msg = "CIE/FDE too large";
       break;
     }
     size += 4;
     if (size > d.size()) {
       msg = "CIE/FDE ends past the end of the section";
       break;
     }
 
     uint64_t off = d.data() - rawData.data();
     pieces.emplace_back(off, this, size, getReloc(off, size, rels, relI));
     d = d.slice(size);
   }
   if (msg)
     errorOrWarn("corrupted .eh_frame: " + Twine(msg) + "\n>>> defined in " +
                 getObjMsg(d.data() - rawData.data()));
 }
 
 static size_t findNull(StringRef s, size_t entSize) {
   for (unsigned i = 0, n = s.size(); i != n; i += entSize) {
     const char *b = s.begin() + i;
     if (std::all_of(b, b + entSize, [](char c) { return c == 0; }))
       return i;
   }
   llvm_unreachable("");
 }
 
 SyntheticSection *MergeInputSection::getParent() const {
   return cast_or_null<SyntheticSection>(parent);
 }
 
 // Split SHF_STRINGS section. Such section is a sequence of
 // null-terminated strings.
 void MergeInputSection::splitStrings(StringRef s, size_t entSize) {
   const bool live = !(flags & SHF_ALLOC) || !config->gcSections;
   const char *p = s.data(), *end = s.data() + s.size();
   if (!std::all_of(end - entSize, end, [](char c) { return c == 0; }))
     fatal(toString(this) + ": string is not null terminated");
   if (entSize == 1) {
     // Optimize the common case.
     do {
       size_t size = strlen(p) + 1;
       pieces.emplace_back(p - s.begin(), xxHash64(StringRef(p, size)), live);
       p += size;
     } while (p != end);
   } else {
     do {
       size_t size = findNull(StringRef(p, end - p), entSize) + entSize;
       pieces.emplace_back(p - s.begin(), xxHash64(StringRef(p, size)), live);
       p += size;
     } while (p != end);
   }
 }
 
 // Split non-SHF_STRINGS section. Such section is a sequence of
 // fixed size records.
 void MergeInputSection::splitNonStrings(ArrayRef<uint8_t> data,
                                         size_t entSize) {
   size_t size = data.size();
   assert((size % entSize) == 0);
   const bool live = !(flags & SHF_ALLOC) || !config->gcSections;
 
   pieces.resize_for_overwrite(size / entSize);
   for (size_t i = 0, j = 0; i != size; i += entSize, j++)
     pieces[j] = {i, (uint32_t)xxHash64(data.slice(i, entSize)), live};
 }
 
 template <class ELFT>
 MergeInputSection::MergeInputSection(ObjFile<ELFT> &f,
                                      const typename ELFT::Shdr &header,
                                      StringRef name)
     : InputSectionBase(f, header, name, InputSectionBase::Merge) {}
 
 MergeInputSection::MergeInputSection(uint64_t flags, uint32_t type,
                                      uint64_t entsize, ArrayRef<uint8_t> data,
                                      StringRef name)
     : InputSectionBase(nullptr, flags, type, entsize, /*Link*/ 0, /*Info*/ 0,
                        /*Alignment*/ entsize, data, name, SectionBase::Merge) {}
 
 // This function is called after we obtain a complete list of input sections
 // that need to be linked. This is responsible to split section contents
 // into small chunks for further processing.
 //
 // Note that this function is called from parallelForEach. This must be
 // thread-safe (i.e. no memory allocation from the pools).
 void MergeInputSection::splitIntoPieces() {
   assert(pieces.empty());
 
   if (flags & SHF_STRINGS)
     splitStrings(toStringRef(data()), entsize);
   else
     splitNonStrings(data(), entsize);
 }
 
 SectionPiece *MergeInputSection::getSectionPiece(uint64_t offset) {
   if (this->data().size() <= offset)
     fatal(toString(this) + ": offset is outside the section");
 
   // If Offset is not at beginning of a section piece, it is not in the map.
   // In that case we need to  do a binary search of the original section piece vector.
   auto it = partition_point(
       pieces, [=](SectionPiece p) { return p.inputOff <= offset; });
   return &it[-1];
 }
 
 // Returns the offset in an output section for a given input offset.
 // Because contents of a mergeable section is not contiguous in output,
 // it is not just an addition to a base output offset.
 uint64_t MergeInputSection::getParentOffset(uint64_t offset) const {
   // If Offset is not at beginning of a section piece, it is not in the map.
   // In that case we need to search from the original section piece vector.
   const SectionPiece &piece = *getSectionPiece(offset);
   uint64_t addend = offset - piece.inputOff;
   return piece.outputOff + addend;
 }
 
 template InputSection::InputSection(ObjFile<ELF32LE> &, const ELF32LE::Shdr &,
                                     StringRef);
 template InputSection::InputSection(ObjFile<ELF32BE> &, const ELF32BE::Shdr &,
                                     StringRef);
 template InputSection::InputSection(ObjFile<ELF64LE> &, const ELF64LE::Shdr &,
                                     StringRef);
 template InputSection::InputSection(ObjFile<ELF64BE> &, const ELF64BE::Shdr &,
                                     StringRef);
 
 template void InputSection::writeTo<ELF32LE>(uint8_t *);
 template void InputSection::writeTo<ELF32BE>(uint8_t *);
 template void InputSection::writeTo<ELF64LE>(uint8_t *);
 template void InputSection::writeTo<ELF64BE>(uint8_t *);
 
 template RelsOrRelas<ELF32LE> InputSectionBase::relsOrRelas<ELF32LE>() const;
 template RelsOrRelas<ELF32BE> InputSectionBase::relsOrRelas<ELF32BE>() const;
 template RelsOrRelas<ELF64LE> InputSectionBase::relsOrRelas<ELF64LE>() const;
 template RelsOrRelas<ELF64BE> InputSectionBase::relsOrRelas<ELF64BE>() const;
 
 template MergeInputSection::MergeInputSection(ObjFile<ELF32LE> &,
                                               const ELF32LE::Shdr &, StringRef);
 template MergeInputSection::MergeInputSection(ObjFile<ELF32BE> &,
                                               const ELF32BE::Shdr &, StringRef);
 template MergeInputSection::MergeInputSection(ObjFile<ELF64LE> &,
                                               const ELF64LE::Shdr &, StringRef);
 template MergeInputSection::MergeInputSection(ObjFile<ELF64BE> &,
                                               const ELF64BE::Shdr &, StringRef);
 
 template EhInputSection::EhInputSection(ObjFile<ELF32LE> &,
                                         const ELF32LE::Shdr &, StringRef);
 template EhInputSection::EhInputSection(ObjFile<ELF32BE> &,
                                         const ELF32BE::Shdr &, StringRef);
 template EhInputSection::EhInputSection(ObjFile<ELF64LE> &,
                                         const ELF64LE::Shdr &, StringRef);
 template EhInputSection::EhInputSection(ObjFile<ELF64BE> &,
                                         const ELF64BE::Shdr &, StringRef);
 
 template void EhInputSection::split<ELF32LE>();
 template void EhInputSection::split<ELF32BE>();
 template void EhInputSection::split<ELF64LE>();
 template void EhInputSection::split<ELF64BE>();
diff --git a/contrib/llvm-project/lld/ELF/Options.td b/contrib/llvm-project/lld/ELF/Options.td
index ca9fdcde791f..cf2013a5f820 100644
--- a/contrib/llvm-project/lld/ELF/Options.td
+++ b/contrib/llvm-project/lld/ELF/Options.td
@@ -1,721 +1,722 @@
 include "llvm/Option/OptParser.td"
 
 // Convenience classes for long options which only accept two dashes. For lld
 // specific or newer long options, we prefer two dashes to avoid collision with
 // short options. For many others, we have to accept both forms to be compatible
 // with GNU ld.
 class FF<string name> : Flag<["--"], name>;
 class JJ<string name>: Joined<["--"], name>;
 
 multiclass EEq<string name, string help> {
   def NAME: Separate<["--"], name>;
   def NAME # _eq: Joined<["--"], name # "=">, Alias<!cast<Separate>(NAME)>,
     HelpText<help>;
 }
 
 multiclass BB<string name, string help1, string help2> {
   def NAME: Flag<["--"], name>, HelpText<help1>;
   def no_ # NAME: Flag<["--"], "no-" # name>, HelpText<help2>;
 }
 
 // For options whose names are multiple letters, either one dash or
 // two can precede the option name except those that start with 'o'.
 class F<string name>: Flag<["--", "-"], name>;
 class J<string name>: Joined<["--", "-"], name>;
 
 multiclass Eq<string name, string help> {
   def NAME: Separate<["--", "-"], name>;
   def NAME # _eq: Joined<["--", "-"], name # "=">, Alias<!cast<Separate>(NAME)>,
     HelpText<help>;
 }
 
 multiclass B<string name, string help1, string help2> {
   def NAME: Flag<["--", "-"], name>, HelpText<help1>;
   def no_ # NAME: Flag<["--", "-"], "no-" # name>, HelpText<help2>;
 }
 
 defm auxiliary: Eq<"auxiliary", "Set DT_AUXILIARY field to the specified name">;
 
 def Bno_symbolic: F<"Bno-symbolic">, HelpText<"Don't bind default visibility defined symbols locally for -shared (default)">;
 
 def Bsymbolic: F<"Bsymbolic">, HelpText<"Bind default visibility defined symbols locally for -shared">;
 
 def Bsymbolic_functions: F<"Bsymbolic-functions">,
   HelpText<"Bind default visibility defined function symbols locally for -shared">;
 
 def Bsymbolic_non_weak_functions: F<"Bsymbolic-non-weak-functions">,
   HelpText<"Bind default visibility defined STB_GLOBAL function symbols locally for -shared">;
 
 def Bdynamic: F<"Bdynamic">, HelpText<"Link against shared libraries (default)">;
 
 def Bstatic: F<"Bstatic">, HelpText<"Do not link against shared libraries">;
 
 def build_id: F<"build-id">, HelpText<"Alias for --build-id=fast">;
 
 def build_id_eq: J<"build-id=">, HelpText<"Generate build ID note">,
   MetaVarName<"[fast,md5,sha1,uuid,0x<hexstring>]">;
 
 defm check_sections: B<"check-sections",
     "Check section addresses for overlaps (default)",
     "Do not check section addresses for overlaps">;
 
 defm compress_debug_sections:
   Eq<"compress-debug-sections", "Compress DWARF debug sections">,
   MetaVarName<"[none,zlib]">;
 
 defm defsym: Eq<"defsym", "Define a symbol alias">, MetaVarName<"<symbol>=<value>">;
 
 defm optimize_bb_jumps: BB<"optimize-bb-jumps",
     "Remove direct jumps at the end to the next basic block",
     "Do not remove any direct jumps at the end to the next basic block (default)">;
 
 defm fortran_common : BB<"fortran-common",
     "Search archive members for definitions to override COMMON symbols (default)",
     "Do not search archive members for definitions to override COMMON symbols">;
 
 defm split_stack_adjust_size
     : Eq<"split-stack-adjust-size",
          "Specify adjustment to stack size when a split-stack function calls a "
          "non-split-stack function">,
       MetaVarName<"<value>">;
 
 def O: JoinedOrSeparate<["-"], "O">, HelpText<"Optimize output file size">;
 
 defm Tbss: Eq<"Tbss", "Same as --section-start with .bss as the sectionname">;
 
 defm Tdata: Eq<"Tdata", "Same as --section-start with .data as the sectionname">;
 
 defm Ttext: Eq<"Ttext", "Same as --section-start with .text as the sectionname">;
 
 def Ttext_segment: Separate<["-", "--"], "Ttext-segment">;
 
 defm allow_multiple_definition: B<"allow-multiple-definition",
     "Allow multiple definitions",
     "Do not allow multiple definitions (default)">;
 
 defm allow_shlib_undefined: B<"allow-shlib-undefined",
     "Allow unresolved references in shared libraries (default when linking a shared library)",
     "Do not allow unresolved references in shared libraries (default when linking an executable)">;
 
 defm apply_dynamic_relocs: BB<"apply-dynamic-relocs",
     "Apply link-time values for dynamic relocations",
     "Do not apply link-time values for dynamic relocations (default)">;
 
 defm dependent_libraries: BB<"dependent-libraries",
     "Process dependent library specifiers from input files (default)",
     "Ignore dependent library specifiers from input files">;
 
 defm as_needed: B<"as-needed",
     "Only set DT_NEEDED for shared libraries if used",
     "Always set DT_NEEDED for shared libraries (default)">;
 
 defm call_graph_ordering_file:
   Eq<"call-graph-ordering-file", "Layout sections to optimize the given callgraph">;
 
 defm call_graph_profile_sort: BB<"call-graph-profile-sort",
     "Reorder sections with call graph profile (default)",
     "Do not reorder sections with call graph profile">;
 
 // --chroot doesn't have a help text because it is an internal option.
 def chroot: Separate<["--"], "chroot">;
 
 defm color_diagnostics: BB<"color-diagnostics",
   "Alias for --color-diagnostics=always",
   "Alias for --color-diagnostics=never">;
 def color_diagnostics_eq: JJ<"color-diagnostics=">,
   HelpText<"Use colors in diagnostics (default: auto)">,
   MetaVarName<"[auto,always,never]">;
 
 def cref: FF<"cref">,
   HelpText<"Output cross reference table. If -Map is specified, print to the map file">;
 
 defm define_common: B<"define-common",
     "Assign space to common symbols",
     "Do not assign space to common symbols">;
 
 defm demangle: B<"demangle",
     "Demangle symbol names (default)",
     "Do not demangle symbol names">;
 
 defm dependency_file: EEq<"dependency-file", "Write a dependency file">,
   MetaVarName<"<file>">;
 
 def disable_new_dtags: F<"disable-new-dtags">,
   HelpText<"Disable new dynamic tags">;
 
 def discard_all: F<"discard-all">, HelpText<"Delete all local symbols">;
 
 def discard_locals: F<"discard-locals">,
   HelpText<"Delete temporary local symbols">;
 
 def discard_none: F<"discard-none">,
   HelpText<"Keep all symbols in the symbol table">;
 
 defm dynamic_linker: Eq<"dynamic-linker", "Which dynamic linker to use">;
 
 defm dynamic_list : Eq<"dynamic-list",
    "Similar to --export-dynamic-symbol-list. When creating a shared object, "
    "this additionally implies -Bsymbolic but does not set DF_SYMBOLIC">,
    MetaVarName<"<file>">;
 
 def eb: F<"EB">, HelpText<"Select the big-endian format in OUTPUT_FORMAT">;
 def el: F<"EL">, HelpText<"Select the little-endian format in OUTPUT_FORMAT">;
 
 defm eh_frame_hdr: B<"eh-frame-hdr",
     "Request creation of .eh_frame_hdr section and PT_GNU_EH_FRAME segment header",
     "Do not create .eh_frame_hdr section">;
 
 def emit_relocs: F<"emit-relocs">, HelpText<"Generate relocations in output">;
 
 def enable_new_dtags: F<"enable-new-dtags">,
   HelpText<"Enable new dynamic tags (default)">;
 
 def end_group: F<"end-group">,
   HelpText<"Ignored for compatibility with GNU unless you pass --warn-backrefs">;
 
 def end_lib: F<"end-lib">,
   HelpText<"End a grouping of objects that should be treated as if they were together in an archive">;
 
 defm entry: Eq<"entry", "Name of entry point symbol">,
   MetaVarName<"<entry>">;
 
 defm error_limit:
   Eq<"error-limit", "Maximum number of errors to emit before stopping (0 = no limit)">;
 
 def error_unresolved_symbols: F<"error-unresolved-symbols">,
   HelpText<"Report unresolved symbols as errors">;
 
 defm error_handling_script: EEq<"error-handling-script",
     "Specify an error handling script">;
 
 defm exclude_libs: Eq<"exclude-libs", "Exclude static libraries from automatic export">;
 
 defm execute_only: BB<"execute-only",
     "Mark executable sections unreadable",
     "Mark executable sections readable (default)">;
 
 defm export_dynamic: B<"export-dynamic",
     "Put symbols in the dynamic symbol table",
     "Do not put symbols in the dynamic symbol table (default)">;
 
 defm export_dynamic_symbol : EEq<"export-dynamic-symbol",
     "(executable) Put matched symbols in the dynamic symbol table. "
     "(shared object) References to matched non-local STV_DEFAULT symbols "
     "shouldn't be bound to definitions within the shared object. "
     "Does not imply -Bsymbolic.">,
     MetaVarName<"glob">;
 
 defm export_dynamic_symbol_list : EEq<"export-dynamic-symbol-list",
    "Read a list of dynamic symbol patterns. Apply --export-dynamic-symbol on each pattern">,
     MetaVarName<"file">;
 
 defm fatal_warnings: B<"fatal-warnings",
     "Treat warnings as errors",
     "Do not treat warnings as errors (default)">;
 
 defm filter: Eq<"filter", "Set DT_FILTER field to the specified name">;
 
 defm fini: Eq<"fini", "Specify a finalizer function">, MetaVarName<"<symbol>">;
 
 def fix_cortex_a53_843419: F<"fix-cortex-a53-843419">,
   HelpText<"Apply fixes for AArch64 Cortex-A53 erratum 843419">;
 
 def fix_cortex_a8: F<"fix-cortex-a8">,
   HelpText<"Apply fixes for ARM Cortex-A8 erratum 657417">;
 
 defm format: Eq<"format", "Change the input format of the inputs following this option">,
   MetaVarName<"[default,elf,binary]">;
 
 defm gc_sections: B<"gc-sections",
     "Enable garbage collection of unused sections",
     "Disable garbage collection of unused sections (default)">;
 
 defm gdb_index: BB<"gdb-index",
     "Generate .gdb_index section",
     "Do not generate .gdb_index section (default)">;
 
 defm gnu_unique: BB<"gnu-unique",
   "Enable STB_GNU_UNIQUE symbol binding (default)",
   "Disable STB_GNU_UNIQUE symbol binding">;
 
 defm hash_style: Eq<"hash-style", "Specify hash style (sysv, gnu or both)">;
 
 def help: F<"help">, HelpText<"Print option help">;
 
 def icf_all: F<"icf=all">, HelpText<"Enable identical code folding">;
 
 def icf_safe: F<"icf=safe">, HelpText<"Enable safe identical code folding">;
 
 def icf_none: F<"icf=none">, HelpText<"Disable identical code folding (default)">;
 
 def ignore_function_address_equality: FF<"ignore-function-address-equality">,
   HelpText<"lld can break the address equality of functions">;
 
 def ignore_data_address_equality: FF<"ignore-data-address-equality">,
   HelpText<"lld can break the address equality of data">;
 
 defm image_base: EEq<"image-base", "Set the base address">;
 
 defm init: Eq<"init", "Specify an initializer function">,
   MetaVarName<"<symbol>">;
 
 defm just_symbols: Eq<"just-symbols", "Just link symbols">;
 
 defm keep_unique: Eq<"keep-unique", "Do not fold this symbol during ICF">;
 
 def library: JoinedOrSeparate<["-"], "l">, MetaVarName<"<libname>">,
   HelpText<"Search for library <libname>">;
 def library_path: JoinedOrSeparate<["-"], "L">, MetaVarName<"<dir>">,
   HelpText<"Add <dir> to the library search path">;
 
 def m: JoinedOrSeparate<["-"], "m">, HelpText<"Set target emulation">;
 
 defm Map: Eq<"Map", "Print a link map to the specified file">;
 
 defm merge_exidx_entries: B<"merge-exidx-entries",
     "Enable merging .ARM.exidx entries (default)",
     "Disable merging .ARM.exidx entries">;
 
 defm mmap_output_file: BB<"mmap-output-file",
     "Mmap the output file for writing (default)",
     "Do not mmap the output file for writing">;
 
 def nmagic: F<"nmagic">, MetaVarName<"<magic>">,
   HelpText<"Do not page align sections, link against static libraries.">;
 
 def nostdlib: F<"nostdlib">,
   HelpText<"Only search directories specified on the command line">;
 
 def no_dynamic_linker: F<"no-dynamic-linker">,
   HelpText<"Inhibit output of .interp section">;
 
 def noinhibit_exec: F<"noinhibit-exec">,
   HelpText<"Retain the executable output file whenever it is still usable">;
 
 def no_nmagic: F<"no-nmagic">, MetaVarName<"<magic>">,
   HelpText<"Page align sections (default)">;
 
 def no_omagic: F<"no-omagic">, MetaVarName<"<magic>">,
   HelpText<"Do not set the text data sections to be writable, page align sections (default)">;
 
 def no_undefined: F<"no-undefined">,
   HelpText<"Report unresolved symbols even if the linker is creating a shared library">;
 
 def o: JoinedOrSeparate<["-"], "o">, MetaVarName<"<path>">,
   HelpText<"Path to file to write output">;
 
 defm oformat: EEq<"oformat", "Specify the binary format for the output object file">,
   MetaVarName<"[elf,binary]">;
 
 def omagic: FF<"omagic">, MetaVarName<"<magic>">,
   HelpText<"Set the text and data sections to be readable and writable, do not page align sections, link against static libraries">;
 
 defm orphan_handling:
   Eq<"orphan-handling", "Control how orphan sections are handled when linker script used">;
 
 defm pack_dyn_relocs:
   EEq<"pack-dyn-relocs", "Pack dynamic relocations in the given format">,
   MetaVarName<"[none,android,relr,android+relr]">;
 
 defm use_android_relr_tags: BB<"use-android-relr-tags",
     "Use SHT_ANDROID_RELR / DT_ANDROID_RELR* tags instead of SHT_RELR / DT_RELR*",
     "Use SHT_RELR / DT_RELR* tags (default)">;
 
 def pic_veneer: F<"pic-veneer">,
   HelpText<"Always generate position independent thunks (veneers)">;
 
 defm pie: B<"pie",
     "Create a position independent executable",
     "Do not create a position independent executable (default)">;
 
 defm print_gc_sections: B<"print-gc-sections",
     "List removed unused sections",
     "Do not list removed unused sections (default)">;
 
 defm print_icf_sections: B<"print-icf-sections",
     "List identical folded sections",
     "Do not list identical folded sections (default)">;
 
 def print_archive_stats: J<"print-archive-stats=">,
   HelpText<"Write archive usage statistics to the specified file. "
            "Print the numbers of members and extracted members for each archive">;
 
 defm print_symbol_order: Eq<"print-symbol-order",
   "Print a symbol order specified by --call-graph-ordering-file into the specified file">;
 
 def pop_state: F<"pop-state">,
   HelpText<"Restore the states saved by --push-state">;
 
 def push_state: F<"push-state">,
   HelpText<"Save the current state of --as-needed, -static and --whole-archive">;
 
 def print_map: F<"print-map">,
   HelpText<"Print a link map to the standard output">;
 
 defm relax: BB<"relax",
   "Enable target-specific relaxations if supported (default)",
   "Disable target-specific relaxations">;
 
 defm reproduce:
   EEq<"reproduce",
      "Write tar file containing inputs and command to reproduce link">;
 
 defm rosegment: BB<"rosegment",
   "Put read-only non-executable sections in their own segment (default)",
   "Do not put read-only non-executable sections in their own segment">;
 
 defm rpath: Eq<"rpath", "Add a DT_RUNPATH to the output">;
 
 def relocatable: F<"relocatable">, HelpText<"Create relocatable object file">;
 
 defm retain_symbols_file:
   Eq<"retain-symbols-file", "Retain only the symbols listed in the file">,
   MetaVarName<"<file>">;
 
 defm script: Eq<"script", "Read linker script">;
 
 defm section_start: Eq<"section-start", "Set address of section">,
   MetaVarName<"<address>">;
 
 def shared: F<"shared">, HelpText<"Build a shared object">;
 
 defm soname: Eq<"soname", "Set DT_SONAME">;
 
 defm sort_section:
   Eq<"sort-section", "Specifies sections sorting rule when linkerscript is used">;
 
 def start_group: F<"start-group">,
   HelpText<"Ignored for compatibility with GNU unless you pass --warn-backrefs">;
 
 def start_lib: F<"start-lib">,
   HelpText<"Start a grouping of objects that should be treated as if they were together in an archive">;
 
 def strip_all: F<"strip-all">, HelpText<"Strip all symbols. Implies --strip-debug">;
 
 def strip_debug: F<"strip-debug">, HelpText<"Strip debugging information">;
 
 defm symbol_ordering_file:
   EEq<"symbol-ordering-file", "Layout sections to place symbols in the order specified by symbol ordering file">;
 
 defm sysroot: Eq<"sysroot", "Set the system root">;
 
 def target1_rel: F<"target1-rel">, HelpText<"Interpret R_ARM_TARGET1 as R_ARM_REL32">;
 
 def target1_abs: F<"target1-abs">, HelpText<"Interpret R_ARM_TARGET1 as R_ARM_ABS32 (default)">;
 
 defm target2:
   Eq<"target2", "Interpret R_ARM_TARGET2 as <type>, where <type> is one of rel, abs, or got-rel">,
   MetaVarName<"<type>">;
 
 defm threads
     : EEq<"threads",
          "Number of threads. '1' disables multi-threading. By default all "
          "available hardware threads are used">;
 
 def time_trace: FF<"time-trace">, HelpText<"Record time trace">;
 def time_trace_file_eq: JJ<"time-trace-file=">, HelpText<"Specify time trace output file">;
 
 defm time_trace_granularity: EEq<"time-trace-granularity",
   "Minimum time granularity (in microseconds) traced by time profiler">;
 
 defm toc_optimize : BB<"toc-optimize",
     "(PowerPC64) Enable TOC related optimizations (default)",
     "(PowerPC64) Disable TOC related optimizations">;
 
 defm pcrel_optimize : BB<"pcrel-optimize",
     "(PowerPC64) Enable PC-relative optimizations (default)",
     "(PowerPC64) Disable PC-relative optimizations">;
 
 def trace: F<"trace">, HelpText<"Print the names of the input files">;
 
 defm trace_symbol: Eq<"trace-symbol", "Trace references to symbols">;
 
 defm undefined: Eq<"undefined", "Force undefined symbol during linking">,
   MetaVarName<"<symbol>">;
 
 defm undefined_glob: EEq<"undefined-glob", "Force undefined symbol during linking">,
   MetaVarName<"<pattern>">;
 
 def unique: F<"unique">, HelpText<"Creates a separate output section for every orphan input section">;
 
 defm unresolved_symbols:
   Eq<"unresolved-symbols", "Determine how to handle unresolved symbols">;
 
 defm undefined_version: B<"undefined-version",
   "Allow unused version in version script (default)",
   "Report version scripts that refer undefined symbols">;
 
 defm rsp_quoting: EEq<"rsp-quoting", "Quoting style for response files">,
   MetaVarName<"[posix,windows]">;
 
 def v: Flag<["-"], "v">, HelpText<"Display the version number">;
 
 def verbose: F<"verbose">, HelpText<"Verbose mode">;
 
 def version: F<"version">, HelpText<"Display the version number and exit">;
 
 def power10_stubs_eq: JJ<"power10-stubs=">, MetaVarName<"<mode>">,
   HelpText<"Whether to use Power10 instructions in call stubs for R_PPC64_REL24_NOTOC and TOC/NOTOC "
   "interworking (yes (default): use; no: don't use). \"auto\" is currently the same as \"yes\"">;
 def power10_stubs: FF<"power10-stubs">, Alias<power10_stubs_eq>, AliasArgs<["yes"]>,
   HelpText<"Alias for --power10-stubs=auto">;
 def no_power10_stubs: FF<"no-power10-stubs">, Alias<power10_stubs_eq>, AliasArgs<["no"]>,
   HelpText<"Alias for --power10-stubs=no">;
 
 defm version_script: Eq<"version-script", "Read a version script">;
 
 defm warn_backrefs: BB<"warn-backrefs",
     "Warn about backward symbol references to extract archive members",
     "Do not warn about backward symbol references to extract archive members (default)">;
 
 defm warn_backrefs_exclude
     : EEq<"warn-backrefs-exclude",
          "Glob describing an archive (or an object file within --start-lib) "
          "which should be ignored for --warn-backrefs.">,
       MetaVarName<"<glob>">;
 
 defm warn_common: B<"warn-common",
     "Warn about duplicate common symbols",
     "Do not warn about duplicate common symbols (default)">;
 
 defm warn_ifunc_textrel: BB<"warn-ifunc-textrel",
     "Warn about using ifunc symbols with text relocations",
     "Do not warn about using ifunc symbols with text relocations (default)">;
 
 defm warn_symbol_ordering: BB<"warn-symbol-ordering",
     "Warn about problems with the symbol ordering file (default)",
     "Do not warn about problems with the symbol ordering file">;
 
 def warn_unresolved_symbols: F<"warn-unresolved-symbols">,
   HelpText<"Report unresolved symbols as warnings">;
 
 defm whole_archive: B<"whole-archive",
     "Force load of all members in a static library",
     "Do not force load of all members in a static library (default)">;
 
 def why_extract: JJ<"why-extract=">, HelpText<"Print to a file about why archive members are extracted">;
 
 defm wrap : Eq<"wrap", "Redirect symbol references to __wrap_symbol and "
                        "__real_symbol references to symbol">,
             MetaVarName<"<symbol>">;
 
 def z: JoinedOrSeparate<["-"], "z">, MetaVarName<"<option>">,
   HelpText<"Linker option extensions">;
 
 def visual_studio_diagnostics_format : FF<"vs-diagnostics">,
 HelpText<"Format diagnostics for Visual Studio compatibility">;
 
 // Aliases
 def: Separate<["-"], "f">, Alias<auxiliary>, HelpText<"Alias for --auxiliary">;
 def: F<"call_shared">, Alias<Bdynamic>, HelpText<"Alias for --Bdynamic">;
 def: F<"dy">, Alias<Bdynamic>, HelpText<"Alias for --Bdynamic">;
 def: F<"dn">, Alias<Bstatic>, HelpText<"Alias for --Bstatic">;
 def: F<"non_shared">, Alias<Bstatic>, HelpText<"Alias for --Bstatic">;
 def: F<"static">, Alias<Bstatic>, HelpText<"Alias for --Bstatic">;
 def: Flag<["-"], "d">, Alias<define_common>, HelpText<"Alias for --define-common">;
 def: F<"dc">, Alias<define_common>, HelpText<"Alias for --define-common">;
 def: F<"dp">, Alias<define_common>, HelpText<"Alias for --define-common">;
 def: Flag<["-"], "x">, Alias<discard_all>, HelpText<"Alias for --discard-all">;
 def: Flag<["-"], "X">, Alias<discard_locals>, HelpText<"Alias for --discard-locals">;
 def: Flag<["-"], "q">, Alias<emit_relocs>, HelpText<"Alias for --emit-relocs">;
 def: Flag<["-"], ")">, Alias<end_group>, HelpText<"Alias for --end-group">;
 def: JoinedOrSeparate<["-"], "e">, Alias<entry>, HelpText<"Alias for --entry">;
 def: Flag<["-"], "E">, Alias<export_dynamic>, HelpText<"Alias for --export-dynamic">;
 def: Separate<["-"], "F">, Alias<filter>, HelpText<"Alias for --filter">;
 def: Separate<["-"], "b">, Alias<format>, HelpText<"Alias for --format">;
 def: Separate<["--", "-"], "library">, Alias<library>;
 def: Joined<["--", "-"], "library=">, Alias<library>;
 def: Separate<["--", "-"], "library-path">, Alias<library_path>;
 def: Joined<["--", "-"], "library-path=">, Alias<library_path>;
 def: Flag<["-"], "n">, Alias<nmagic>, HelpText<"Alias for --nmagic">;
 def: Flag<["-"], "N">, Alias<omagic>, HelpText<"Alias for --omagic">;
 def: Joined<["--"], "output=">, Alias<o>, HelpText<"Alias for -o">;
 def: Separate<["--"], "output">, Alias<o>, HelpText<"Alias for -o">;
 def: F<"pic-executable">, Alias<pie>, HelpText<"Alias for --pie">;
 def: Flag<["-"], "M">, Alias<print_map>, HelpText<"Alias for --print-map">;
 def: Flag<["-"], "r">, Alias<relocatable>, HelpText<"Alias for --relocatable">;
 def: JoinedOrSeparate<["-"], "R">, Alias<rpath>, HelpText<"Alias for --rpath">;
 def: JoinedOrSeparate<["-"], "T">, Alias<script>, HelpText<"Alias for --script">;
 def: F<"Bshareable">, Alias<shared>, HelpText<"Alias for --shared">;
 def: JoinedOrSeparate<["-"], "h">, Alias<soname>, HelpText<"Alias for --soname">;
 def: Flag<["-"], "(">, Alias<start_group>, HelpText<"Alias for --start-group">;
 def: Flag<["-"], "s">, Alias<strip_all>, HelpText<"Alias for --strip-all">;
 def: Flag<["-"], "S">, Alias<strip_debug>, HelpText<"Alias for --strip-debug">;
 def: Flag<["-"], "t">, Alias<trace>, HelpText<"Alias for --trace">;
 def: Joined<["-", "--"], "Ttext-segment=">, Alias<Ttext_segment>;
 def: JoinedOrSeparate<["-"], "y">, Alias<trace_symbol>, HelpText<"Alias for --trace-symbol">;
 def: JoinedOrSeparate<["-"], "u">, Alias<undefined>, HelpText<"Alias for --undefined">;
 def: Flag<["-"], "V">, Alias<version>, HelpText<"Alias for --version">;
 
 // LTO-related options.
 def lto_aa_pipeline: JJ<"lto-aa-pipeline=">,
   HelpText<"AA pipeline to run during LTO. Used in conjunction with -lto-newpm-passes">;
 def lto_debug_pass_manager: FF<"lto-debug-pass-manager">,
   HelpText<"Debug new pass manager">;
 def lto_emit_asm: FF<"lto-emit-asm">,
   HelpText<"Emit assembly code">;
 defm lto_legacy_pass_manager: BB<"lto-legacy-pass-manager",
   "Use the legacy pass manager in LLVM",
   "Use the new pass manager in LLVM">;
 def lto_newpm_passes: JJ<"lto-newpm-passes=">,
   HelpText<"Passes to run during LTO">;
 def lto_O: JJ<"lto-O">, MetaVarName<"<opt-level>">,
   HelpText<"Optimization level for LTO">;
 def lto_partitions: JJ<"lto-partitions=">,
   HelpText<"Number of LTO codegen partitions">;
 def lto_cs_profile_generate: FF<"lto-cs-profile-generate">,
   HelpText<"Perform context sensitive PGO instrumentation">;
 def lto_cs_profile_file: JJ<"lto-cs-profile-file=">,
   HelpText<"Context sensitive profile file path">;
 defm lto_pgo_warn_mismatch: BB<"lto-pgo-warn-mismatch",
   "turn on warnings about profile cfg mismatch (default)>",
   "turn off warnings about profile cfg mismatch">;
 def lto_obj_path_eq: JJ<"lto-obj-path=">;
 def lto_sample_profile: JJ<"lto-sample-profile=">,
   HelpText<"Sample profile file path">;
 defm lto_whole_program_visibility: BB<"lto-whole-program-visibility",
   "Asserts that the LTO link has whole program visibility",
   "Asserts that the LTO link does not have whole program visibility">;
 def disable_verify: F<"disable-verify">;
 defm mllvm: Eq<"mllvm", "Additional arguments to forward to LLVM's option processing">;
 def opt_remarks_filename: Separate<["--"], "opt-remarks-filename">,
   HelpText<"YAML output file for optimization remarks">;
 defm opt_remarks_hotness_threshold: EEq<"opt-remarks-hotness-threshold",
   "Minimum profile count required for an optimization remark to be output."
   " Use 'auto' to apply the threshold from profile summary.">,
   MetaVarName<"<value>">;
 def opt_remarks_passes: Separate<["--"], "opt-remarks-passes">,
   HelpText<"Regex for the passes that need to be serialized to the output file">;
 def opt_remarks_with_hotness: FF<"opt-remarks-with-hotness">,
   HelpText<"Include hotness information in the optimization remarks file">;
 def opt_remarks_format: Separate<["--"], "opt-remarks-format">,
   HelpText<"The format used for serializing remarks (default: YAML)">;
 def save_temps: F<"save-temps">, HelpText<"Save intermediate LTO compilation results">;
 def lto_basic_block_sections: JJ<"lto-basic-block-sections=">,
   HelpText<"Enable basic block sections for LTO">;
 defm lto_unique_basic_block_section_names: BB<"lto-unique-basic-block-section-names",
     "Give unique names to every basic block section for LTO",
     "Do not give unique names to every basic block section for LTO (default)">;
 defm shuffle_sections: EEq<"shuffle-sections",
   "Shuffle matched sections using the given seed before mapping them to the output sections. "
   "If -1, reverse the section order. If 0, use a random seed">,
   MetaVarName<"<section-glob>=<seed>">;
 def thinlto_cache_dir: JJ<"thinlto-cache-dir=">,
   HelpText<"Path to ThinLTO cached object file directory">;
 defm thinlto_cache_policy: EEq<"thinlto-cache-policy", "Pruning policy for the ThinLTO cache">;
 def thinlto_emit_imports_files: FF<"thinlto-emit-imports-files">;
 def thinlto_index_only: FF<"thinlto-index-only">;
 def thinlto_index_only_eq: JJ<"thinlto-index-only=">;
 def thinlto_jobs: JJ<"thinlto-jobs=">,
   HelpText<"Number of ThinLTO jobs. Default to --threads=">;
 def thinlto_object_suffix_replace_eq: JJ<"thinlto-object-suffix-replace=">;
 def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">;
 def thinlto_single_module_eq: JJ<"thinlto-single-module=">,
   HelpText<"Specific a single module to compile in ThinLTO mode, for debugging only">;
 
 def: J<"plugin-opt=O">, Alias<lto_O>, HelpText<"Alias for --lto-O">;
 def: F<"plugin-opt=debug-pass-manager">,
   Alias<lto_debug_pass_manager>, HelpText<"Alias for --lto-debug-pass-manager">;
 def: F<"plugin-opt=disable-verify">, Alias<disable_verify>, HelpText<"Alias for --disable-verify">;
 def plugin_opt_dwo_dir_eq: J<"plugin-opt=dwo_dir=">,
   HelpText<"Directory to store .dwo files when LTO and debug fission are used">;
 def plugin_opt_emit_asm: F<"plugin-opt=emit-asm">,
   Alias<lto_emit_asm>, HelpText<"Alias for --lto-emit-asm">;
 def plugin_opt_emit_llvm: F<"plugin-opt=emit-llvm">;
 def: J<"plugin-opt=jobs=">, Alias<thinlto_jobs>, HelpText<"Alias for --thinlto-jobs">;
 def: J<"plugin-opt=lto-partitions=">, Alias<lto_partitions>, HelpText<"Alias for --lto-partitions">;
 def plugin_opt_mcpu_eq: J<"plugin-opt=mcpu=">;
 def: F<"plugin-opt=new-pass-manager">,
   Alias<no_lto_legacy_pass_manager>, HelpText<"Alias for --no-lto-legacy-pass-manager">;
 def: F<"plugin-opt=legacy-pass-manager">,
   Alias<lto_legacy_pass_manager>, HelpText<"Alias for --no-legacy-pass-manager">;
 def: F<"plugin-opt=cs-profile-generate">,
   Alias<lto_cs_profile_generate>, HelpText<"Alias for --lto-cs-profile-generate">;
 def: J<"plugin-opt=cs-profile-path=">,
   Alias<lto_cs_profile_file>, HelpText<"Alias for --lto-cs-profile-file">;
 def: J<"plugin-opt=obj-path=">,
   Alias<lto_obj_path_eq>,
   HelpText<"Alias for --lto-obj-path=">;
 def: J<"plugin-opt=opt-remarks-filename=">,
   Alias<opt_remarks_filename>,
   HelpText<"Alias for --opt-remarks-filename">;
 def: J<"plugin-opt=opt-remarks-passes=">,
   Alias<opt_remarks_passes>,
   HelpText<"Alias for --opt-remarks-passes">;
 def: J<"plugin-opt=opt-remarks-format=">,
   Alias<opt_remarks_format>,
   HelpText<"Alias for --opt-remarks-format">;
 def: F<"plugin-opt=opt-remarks-with-hotness">,
   Alias<opt_remarks_with_hotness>,
   HelpText<"Alias for --opt-remarks-with_hotness">;
 def: J<"plugin-opt=opt-remarks-hotness-threshold=">,
   Alias<opt_remarks_hotness_threshold>,
   HelpText<"Alias for --opt-remarks-hotness-threshold">;
 def: J<"plugin-opt=sample-profile=">,
   Alias<lto_sample_profile>, HelpText<"Alias for --lto-sample-profile">;
 def: F<"plugin-opt=save-temps">, Alias<save_temps>, HelpText<"Alias for --save-temps">;
 def: F<"plugin-opt=thinlto-emit-imports-files">,
   Alias<thinlto_emit_imports_files>,
   HelpText<"Alias for --thinlto-emit-imports-files">;
 def: F<"plugin-opt=thinlto-index-only">,
   Alias<thinlto_index_only>,
   HelpText<"Alias for --thinlto-index-only">;
 def: J<"plugin-opt=thinlto-index-only=">,
   Alias<thinlto_index_only_eq>,
   HelpText<"Alias for --thinlto-index-only=">;
 def: J<"plugin-opt=thinlto-object-suffix-replace=">,
  Alias<thinlto_object_suffix_replace_eq>,
  HelpText<"Alias for --thinlto-object-suffix-replace=">;
 def: J<"plugin-opt=thinlto-prefix-replace=">,
   Alias<thinlto_prefix_replace_eq>,
   HelpText<"Alias for --thinlto-prefix-replace=">;
 
 // Ignore LTO plugin-related options.
 // clang -flto passes -plugin and -plugin-opt to the linker. This is required
 // for ld.gold and ld.bfd to get LTO working. But it's not for lld which doesn't
 // rely on a plugin. Instead of detecting which linker is used on clang side we
 // just ignore the option on lld side as it's easier. In fact, the linker could
 // be called 'ld' and understanding which linker is used would require parsing of
 // --version output.
 defm plugin: Eq<"plugin", "Ignored for compatibility with GNU linkers">;
 
 def plugin_opt_eq_minus: J<"plugin-opt=-">,
   HelpText<"Specify an LLVM option for compatibility with LLVMgold.so">;
 def: J<"plugin-opt=thinlto">;
 
 // Ignore GCC collect2 LTO plugin related options. Note that we don't support
 // GCC LTO, but GCC collect2 passes these options even in non-LTO mode.
 def: J<"plugin-opt=-fresolution=">;
 def: J<"plugin-opt=-pass-through=">;
 // This may be either an unhandled LLVMgold.so feature or GCC passed
 // -plugin-opt=path/to/{liblto_plugin.so,lto-wrapper}
 def plugin_opt_eq : J<"plugin-opt=">;
 
 // Options listed below are silently ignored for now for compatibility.
 def: F<"detect-odr-violations">;
 def: Flag<["-"], "g">;
 def: F<"long-plt">;
+def: FF<"no-add-needed">;
 def: F<"no-copy-dt-needed-entries">;
 def: F<"no-ctors-in-init-array">;
 def: F<"no-keep-memory">;
 def: F<"no-warn-mismatch">;
 def: Separate<["--", "-"], "rpath-link">;
 def: J<"rpath-link=">;
 def: F<"secure-plt">;
 def: F<"sort-common">;
 def: F<"stats">;
 def: F<"warn-execstack">;
 def: F<"warn-once">;
 def: F<"warn-shared-textrel">;
 def: JoinedOrSeparate<["-"], "G">;
 
 // Hidden option used for testing MIPS multi-GOT implementation.
 defm mips_got_size:
   Eq<"mips-got-size", "Max size of a single MIPS GOT. 0x10000 by default.">,
   Flags<[HelpHidden]>;
 
 // Hidden option used to opt-in to additional output checks.
 defm check_dynamic_relocations: BB<"check-dynamic-relocations",
     "Perform additional validation of the written dynamic relocations",
     "Do not perform additional validation of the written dynamic relocations">,
   Flags<[HelpHidden]>;
diff --git a/contrib/llvm-project/llvm/include/llvm/Support/AArch64TargetParser.def b/contrib/llvm-project/llvm/include/llvm/Support/AArch64TargetParser.def
index a953e9439db4..c45c149c6084 100644
--- a/contrib/llvm-project/llvm/include/llvm/Support/AArch64TargetParser.def
+++ b/contrib/llvm-project/llvm/include/llvm/Support/AArch64TargetParser.def
@@ -1,295 +1,297 @@
 //===- AARCH64TargetParser.def - AARCH64 target parsing defines ---------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file provides defines to build up the AARCH64 target parser's logic.
 //
 //===----------------------------------------------------------------------===//
 
 // NOTE: NO INCLUDE GUARD DESIRED!
 
 #ifndef AARCH64_ARCH
 #define AARCH64_ARCH(NAME, ID, CPU_ATTR, SUB_ARCH, ARCH_ATTR, ARCH_FPU, ARCH_BASE_EXT)
 #endif
 AARCH64_ARCH("invalid", INVALID, "", "",
              ARMBuildAttrs::CPUArch::v8_A, FK_NONE, AArch64::AEK_NONE)
 AARCH64_ARCH("armv8-a", ARMV8A, "8-A", "v8", ARMBuildAttrs::CPUArch::v8_A,
              FK_CRYPTO_NEON_FP_ARMV8,
              (AArch64::AEK_CRYPTO | AArch64::AEK_FP | AArch64::AEK_SIMD))
 AARCH64_ARCH("armv8.1-a", ARMV8_1A, "8.1-A", "v8.1a",
              ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8,
              (AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP |
               AArch64::AEK_SIMD | AArch64::AEK_LSE | AArch64::AEK_RDM))
 AARCH64_ARCH("armv8.2-a", ARMV8_2A, "8.2-A", "v8.2a",
              ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8,
              (AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP |
               AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE |
               AArch64::AEK_RDM))
 AARCH64_ARCH("armv8.3-a", ARMV8_3A, "8.3-A", "v8.3a",
              ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8,
              (AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP |
               AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE |
               AArch64::AEK_RDM | AArch64::AEK_RCPC))
 AARCH64_ARCH("armv8.4-a", ARMV8_4A, "8.4-A", "v8.4a",
              ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8,
              (AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP |
               AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE |
               AArch64::AEK_RDM | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD))
 AARCH64_ARCH("armv8.5-a", ARMV8_5A, "8.5-A", "v8.5a",
              ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8,
              (AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP |
               AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE |
               AArch64::AEK_RDM | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD))
 AARCH64_ARCH("armv8.6-a", ARMV8_6A, "8.6-A", "v8.6a",
              ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8,
              (AArch64::AEK_CRC  | AArch64::AEK_FP   |
               AArch64::AEK_SIMD | AArch64::AEK_RAS  | AArch64::AEK_LSE     |
               AArch64::AEK_RDM  | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD |
               AArch64::AEK_SM4  | AArch64::AEK_SHA3 | AArch64::AEK_BF16    |
               AArch64::AEK_SHA2 | AArch64::AEK_AES  | AArch64::AEK_I8MM))
 AARCH64_ARCH("armv8.7-a", ARMV8_7A, "8.7-A", "v8.7a",
              ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8,
              (AArch64::AEK_CRC | AArch64::AEK_FP |
               AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE |
               AArch64::AEK_RDM | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD |
               AArch64::AEK_SM4 | AArch64::AEK_SHA3 | AArch64::AEK_BF16 |
               AArch64::AEK_SHA2 | AArch64::AEK_AES | AArch64::AEK_I8MM))
 AARCH64_ARCH("armv8.8-a", ARMV8_8A, "8.8-A", "v8.8a",
              ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8,
              (AArch64::AEK_CRC | AArch64::AEK_FP |
               AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE |
               AArch64::AEK_RDM | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD |
               AArch64::AEK_SM4 | AArch64::AEK_SHA3 | AArch64::AEK_BF16 |
               AArch64::AEK_SHA2 | AArch64::AEK_AES | AArch64::AEK_I8MM))
 AARCH64_ARCH("armv9-a",   ARMV9A, "9-A", "v9a",
              ARMBuildAttrs::CPUArch::v8_A, FK_NEON_FP_ARMV8,
              (AArch64::AEK_CRC | AArch64::AEK_FP |
               AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE |
               AArch64::AEK_RDM | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD |
               AArch64::AEK_SVE2))
 AARCH64_ARCH("armv9.1-a", ARMV9_1A, "9.1-A", "v9.1a",
              ARMBuildAttrs::CPUArch::v8_A, FK_NEON_FP_ARMV8,
              (AArch64::AEK_CRC | AArch64::AEK_FP |
               AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE |
               AArch64::AEK_RDM | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD |
               AArch64::AEK_SVE2))
 AARCH64_ARCH("armv9.2-a", ARMV9_2A, "9.2-A", "v9.2a",
              ARMBuildAttrs::CPUArch::v8_A, FK_NEON_FP_ARMV8,
              (AArch64::AEK_CRC | AArch64::AEK_FP |
               AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE |
               AArch64::AEK_RDM | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD |
               AArch64::AEK_SVE2))
 AARCH64_ARCH("armv9.3-a", ARMV9_3A, "9.3-A", "v9.3",
              ARMBuildAttrs::CPUArch::v8_A, FK_CRYPTO_NEON_FP_ARMV8,
              (AArch64::AEK_CRC | AArch64::AEK_FP |
               AArch64::AEK_SIMD | AArch64::AEK_RAS | AArch64::AEK_LSE |
               AArch64::AEK_RDM | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD |
               AArch64::AEK_SVE2))
 // For v8-R, we do not enable crypto and align with GCC that enables a more
 // minimal set of optional architecture extensions.
 AARCH64_ARCH("armv8-r", ARMV8R, "8-R", "v8r",
              ARMBuildAttrs::CPUArch::v8_R, FK_CRYPTO_NEON_FP_ARMV8,
              (AArch64::AEK_CRC     | AArch64::AEK_RDM  | AArch64::AEK_SSBS |
               AArch64::AEK_DOTPROD | AArch64::AEK_FP   | AArch64::AEK_SIMD |
               AArch64::AEK_FP16    | AArch64::AEK_FP16FML | AArch64::AEK_RAS |
               AArch64::AEK_RCPC    | AArch64::AEK_SB))
 #undef AARCH64_ARCH
 
 #ifndef AARCH64_ARCH_EXT_NAME
 #define AARCH64_ARCH_EXT_NAME(NAME, ID, FEATURE, NEGFEATURE)
 #endif
 // FIXME: This would be nicer were it tablegen
 AARCH64_ARCH_EXT_NAME("invalid",      AArch64::AEK_INVALID,     nullptr,         nullptr)
 AARCH64_ARCH_EXT_NAME("none",         AArch64::AEK_NONE,        nullptr,         nullptr)
 AARCH64_ARCH_EXT_NAME("crc",          AArch64::AEK_CRC,         "+crc",          "-crc")
 AARCH64_ARCH_EXT_NAME("lse",          AArch64::AEK_LSE,         "+lse",          "-lse")
 AARCH64_ARCH_EXT_NAME("rdm",          AArch64::AEK_RDM,         "+rdm",          "-rdm")
 AARCH64_ARCH_EXT_NAME("crypto",       AArch64::AEK_CRYPTO,      "+crypto",       "-crypto")
 AARCH64_ARCH_EXT_NAME("sm4",          AArch64::AEK_SM4,         "+sm4",          "-sm4")
 AARCH64_ARCH_EXT_NAME("sha3",         AArch64::AEK_SHA3,        "+sha3",         "-sha3")
 AARCH64_ARCH_EXT_NAME("sha2",         AArch64::AEK_SHA2,        "+sha2",         "-sha2")
 AARCH64_ARCH_EXT_NAME("aes",          AArch64::AEK_AES,         "+aes",          "-aes")
 AARCH64_ARCH_EXT_NAME("dotprod",      AArch64::AEK_DOTPROD,     "+dotprod",      "-dotprod")
 AARCH64_ARCH_EXT_NAME("fp",           AArch64::AEK_FP,          "+fp-armv8",     "-fp-armv8")
 AARCH64_ARCH_EXT_NAME("simd",         AArch64::AEK_SIMD,        "+neon",         "-neon")
 AARCH64_ARCH_EXT_NAME("fp16",         AArch64::AEK_FP16,        "+fullfp16",     "-fullfp16")
 AARCH64_ARCH_EXT_NAME("fp16fml",      AArch64::AEK_FP16FML,     "+fp16fml",      "-fp16fml")
 AARCH64_ARCH_EXT_NAME("profile",      AArch64::AEK_PROFILE,     "+spe",          "-spe")
 AARCH64_ARCH_EXT_NAME("ras",          AArch64::AEK_RAS,         "+ras",          "-ras")
 AARCH64_ARCH_EXT_NAME("sve",          AArch64::AEK_SVE,         "+sve",          "-sve")
 AARCH64_ARCH_EXT_NAME("sve2",         AArch64::AEK_SVE2,        "+sve2",         "-sve2")
 AARCH64_ARCH_EXT_NAME("sve2-aes",     AArch64::AEK_SVE2AES,     "+sve2-aes",     "-sve2-aes")
 AARCH64_ARCH_EXT_NAME("sve2-sm4",     AArch64::AEK_SVE2SM4,     "+sve2-sm4",     "-sve2-sm4")
 AARCH64_ARCH_EXT_NAME("sve2-sha3",    AArch64::AEK_SVE2SHA3,    "+sve2-sha3",    "-sve2-sha3")
 AARCH64_ARCH_EXT_NAME("sve2-bitperm", AArch64::AEK_SVE2BITPERM, "+sve2-bitperm", "-sve2-bitperm")
 AARCH64_ARCH_EXT_NAME("rcpc",         AArch64::AEK_RCPC,        "+rcpc",         "-rcpc")
 AARCH64_ARCH_EXT_NAME("rng",          AArch64::AEK_RAND,        "+rand",         "-rand")
 AARCH64_ARCH_EXT_NAME("memtag",       AArch64::AEK_MTE,         "+mte",          "-mte")
 AARCH64_ARCH_EXT_NAME("ssbs",         AArch64::AEK_SSBS,        "+ssbs",         "-ssbs")
 AARCH64_ARCH_EXT_NAME("sb",           AArch64::AEK_SB,          "+sb",           "-sb")
 AARCH64_ARCH_EXT_NAME("predres",      AArch64::AEK_PREDRES,     "+predres",      "-predres")
 AARCH64_ARCH_EXT_NAME("bf16",         AArch64::AEK_BF16,        "+bf16",         "-bf16")
 AARCH64_ARCH_EXT_NAME("i8mm",         AArch64::AEK_I8MM,        "+i8mm",         "-i8mm")
 AARCH64_ARCH_EXT_NAME("f32mm",        AArch64::AEK_F32MM,       "+f32mm",        "-f32mm")
 AARCH64_ARCH_EXT_NAME("f64mm",        AArch64::AEK_F64MM,       "+f64mm",        "-f64mm")
 AARCH64_ARCH_EXT_NAME("tme",          AArch64::AEK_TME,         "+tme",          "-tme")
 AARCH64_ARCH_EXT_NAME("ls64",         AArch64::AEK_LS64,        "+ls64",         "-ls64")
 AARCH64_ARCH_EXT_NAME("brbe",         AArch64::AEK_BRBE,        "+brbe",         "-brbe")
 AARCH64_ARCH_EXT_NAME("pauth",        AArch64::AEK_PAUTH,       "+pauth",        "-pauth")
 AARCH64_ARCH_EXT_NAME("flagm",        AArch64::AEK_FLAGM,       "+flagm",        "-flagm")
 AARCH64_ARCH_EXT_NAME("sme",          AArch64::AEK_SME,         "+sme",          "-sme")
 AARCH64_ARCH_EXT_NAME("sme-f64",      AArch64::AEK_SMEF64,      "+sme-f64",      "-sme-f64")
 AARCH64_ARCH_EXT_NAME("sme-i64",      AArch64::AEK_SMEI64,      "+sme-i64",      "-sme-i64")
 AARCH64_ARCH_EXT_NAME("hbc",          AArch64::AEK_HBC,         "+hbc",          "-hbc")
 AARCH64_ARCH_EXT_NAME("mops",         AArch64::AEK_MOPS,        "+mops",         "-mops")
 AARCH64_ARCH_EXT_NAME("pmuv3",        AArch64::AEK_PERFMON,     "+perfmon",      "-perfmon")
 #undef AARCH64_ARCH_EXT_NAME
 
 #ifndef AARCH64_CPU_NAME
 #define AARCH64_CPU_NAME(NAME, ID, DEFAULT_FPU, IS_DEFAULT, DEFAULT_EXT)
 #endif
 AARCH64_CPU_NAME("cortex-a34", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_CRC))
 AARCH64_CPU_NAME("cortex-a35", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_CRC))
 AARCH64_CPU_NAME("cortex-a53", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, true,
                  (AArch64::AEK_CRC))
 AARCH64_CPU_NAME("cortex-a55", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC))
 AARCH64_CPU_NAME("cortex-a510", ARMV9A, FK_NEON_FP_ARMV8, false,
                  (AArch64::AEK_BF16 | AArch64::AEK_I8MM | AArch64::AEK_SB |
                   AArch64::AEK_PAUTH | AArch64::AEK_MTE | AArch64::AEK_SSBS |
                   AArch64::AEK_SVE | AArch64::AEK_SVE2 | AArch64::AEK_SVE2BITPERM |
                   AArch64::AEK_FP16FML))
 AARCH64_CPU_NAME("cortex-a57", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_CRC))
 AARCH64_CPU_NAME("cortex-a65", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | AArch64::AEK_RAS |
                   AArch64::AEK_RCPC | AArch64::AEK_SSBS))
 AARCH64_CPU_NAME("cortex-a65ae", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | AArch64::AEK_RAS |
                   AArch64::AEK_RCPC | AArch64::AEK_SSBS))
 AARCH64_CPU_NAME("cortex-a72", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_CRC))
 AARCH64_CPU_NAME("cortex-a73", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_CRC))
 AARCH64_CPU_NAME("cortex-a75", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC))
 AARCH64_CPU_NAME("cortex-a76", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC |
                   AArch64::AEK_SSBS))
 AARCH64_CPU_NAME("cortex-a76ae", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC |
                   AArch64::AEK_SSBS))
 AARCH64_CPU_NAME("cortex-a77", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD |
                   AArch64::AEK_SSBS))
 AARCH64_CPU_NAME("cortex-a78", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC |
                   AArch64::AEK_SSBS))
 AARCH64_CPU_NAME("cortex-a78c", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC |
                   AArch64::AEK_SSBS))
 AARCH64_CPU_NAME("cortex-a710", ARMV9A, FK_NEON_FP_ARMV8, false,
                  (AArch64::AEK_MTE | AArch64::AEK_PAUTH | AArch64::AEK_FLAGM |
                   AArch64::AEK_SB | AArch64::AEK_I8MM | AArch64::AEK_FP16FML |
                   AArch64::AEK_SVE | AArch64::AEK_SVE2 | AArch64::AEK_SVE2BITPERM |
                   AArch64::AEK_BF16))
 AARCH64_CPU_NAME("cortex-r82", ARMV8R, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_LSE))
 AARCH64_CPU_NAME("cortex-x1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC |
                   AArch64::AEK_SSBS))
 AARCH64_CPU_NAME("cortex-x1c", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC |
                   AArch64::AEK_SSBS | AArch64::AEK_PAUTH))
 AARCH64_CPU_NAME("cortex-x2", ARMV9A, FK_NEON_FP_ARMV8, false,
                  (AArch64::AEK_MTE | AArch64::AEK_BF16 | AArch64::AEK_I8MM |
                   AArch64::AEK_PAUTH | AArch64::AEK_SSBS | AArch64::AEK_SB |
                   AArch64::AEK_SVE | AArch64::AEK_SVE2 | AArch64::AEK_SVE2BITPERM |
                   AArch64::AEK_FP16FML))
 AARCH64_CPU_NAME("neoverse-e1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 | AArch64::AEK_RAS |
                   AArch64::AEK_RCPC | AArch64::AEK_SSBS))
 AARCH64_CPU_NAME("neoverse-n1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_DOTPROD | AArch64::AEK_FP16 |
                   AArch64::AEK_PROFILE | AArch64::AEK_RAS | AArch64::AEK_RCPC |
                   AArch64::AEK_SSBS))
 AARCH64_CPU_NAME("neoverse-n2", ARMV8_5A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_BF16 | AArch64::AEK_DOTPROD | AArch64::AEK_FP16 |
                   AArch64::AEK_I8MM | AArch64::AEK_MTE | AArch64::AEK_RAS |
                   AArch64::AEK_RCPC | AArch64::AEK_SB | AArch64::AEK_SSBS |
                   AArch64::AEK_SVE | AArch64::AEK_SVE2 | AArch64::AEK_SVE2BITPERM))
 AARCH64_CPU_NAME("neoverse-512tvb", ARMV8_4A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_RAS | AArch64::AEK_SVE | AArch64::AEK_SSBS |
                   AArch64::AEK_RCPC | AArch64::AEK_FP16 | AArch64::AEK_BF16 |
                   AArch64::AEK_DOTPROD ))
 AARCH64_CPU_NAME("neoverse-v1", ARMV8_4A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_RAS | AArch64::AEK_SVE | AArch64::AEK_SSBS |
                   AArch64::AEK_RCPC | AArch64::AEK_FP16 | AArch64::AEK_BF16 |
                   AArch64::AEK_DOTPROD ))
 AARCH64_CPU_NAME("cyclone", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_NONE))
 AARCH64_CPU_NAME("apple-a7", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_NONE))
 AARCH64_CPU_NAME("apple-a8", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_NONE))
 AARCH64_CPU_NAME("apple-a9", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_NONE))
 AARCH64_CPU_NAME("apple-a10", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_CRC | AArch64::AEK_RDM))
 AARCH64_CPU_NAME("apple-a11", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16))
 AARCH64_CPU_NAME("apple-a12", ARMV8_3A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16))
 AARCH64_CPU_NAME("apple-a13", ARMV8_4A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_FP16FML))
 AARCH64_CPU_NAME("apple-a14", ARMV8_5A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_FP16FML))
 AARCH64_CPU_NAME("apple-m1", ARMV8_5A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_FP16FML))
 AARCH64_CPU_NAME("apple-s4", ARMV8_3A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16))
 AARCH64_CPU_NAME("apple-s5", ARMV8_3A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16))
 AARCH64_CPU_NAME("exynos-m3", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_CRC))
 AARCH64_CPU_NAME("exynos-m4", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_DOTPROD | AArch64::AEK_FP16))
 AARCH64_CPU_NAME("exynos-m5", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_DOTPROD | AArch64::AEK_FP16))
 AARCH64_CPU_NAME("falkor", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_CRC | AArch64::AEK_RDM))
 AARCH64_CPU_NAME("saphira", ARMV8_3A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_PROFILE))
 AARCH64_CPU_NAME("kryo", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_CRC))
 AARCH64_CPU_NAME("thunderx2t99", ARMV8_1A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_NONE))
 AARCH64_CPU_NAME("thunderx3t110", ARMV8_3A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_CRC | AEK_CRYPTO | AEK_FP | AEK_SIMD |
                   AEK_LSE | AEK_RAND | AArch64::AEK_PROFILE |
                   AArch64::AEK_RAS))
 AARCH64_CPU_NAME("thunderx", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_CRC | AArch64::AEK_PROFILE))
 AARCH64_CPU_NAME("thunderxt88", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_CRC | AArch64::AEK_PROFILE))
 AARCH64_CPU_NAME("thunderxt81", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_CRC | AArch64::AEK_PROFILE))
 AARCH64_CPU_NAME("thunderxt83", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_CRC | AArch64::AEK_PROFILE))
 AARCH64_CPU_NAME("tsv110", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_DOTPROD |
                   AArch64::AEK_FP16 | AArch64::AEK_FP16FML |
                   AArch64::AEK_PROFILE))
 AARCH64_CPU_NAME("a64fx", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_SVE))
 AARCH64_CPU_NAME("carmel", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  AArch64::AEK_FP16)
+AARCH64_CPU_NAME("ampere1", ARMV8_6A, FK_CRYPTO_NEON_FP_ARMV8, false,
+                 (AArch64::AEK_FP16 | AArch64::AEK_SB | AArch64::AEK_SSBS))
 // Invalid CPU
 AARCH64_CPU_NAME("invalid", INVALID, FK_INVALID, true, AArch64::AEK_INVALID)
 #undef AARCH64_CPU_NAME
diff --git a/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 6d415c9c7f90..847df84afba6 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/contrib/llvm-project/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1,3516 +1,3521 @@
 //===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
 /// This file implements the IRTranslator class.
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 #include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RuntimeLibcalls.h"
 #include "llvm/CodeGen/StackProtector.h"
 #include "llvm/CodeGen/SwitchLoweringUtils.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GetElementPtrTypeIterator.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/User.h"
 #include "llvm/IR/Value.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/LowLevelTypeImpl.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetIntrinsicInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/Utils/MemoryOpRemark.h"
 #include <algorithm>
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
 #include <iterator>
 #include <string>
 #include <utility>
 #include <vector>
 
 #define DEBUG_TYPE "irtranslator"
 
 using namespace llvm;
 
 static cl::opt<bool>
     EnableCSEInIRTranslator("enable-cse-in-irtranslator",
                             cl::desc("Should enable CSE in irtranslator"),
                             cl::Optional, cl::init(false));
 char IRTranslator::ID = 0;
 
 INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
                 false, false)
 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
 INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(StackProtector)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
                 false, false)
 
 static void reportTranslationError(MachineFunction &MF,
                                    const TargetPassConfig &TPC,
                                    OptimizationRemarkEmitter &ORE,
                                    OptimizationRemarkMissed &R) {
   MF.getProperties().set(MachineFunctionProperties::Property::FailedISel);
 
   // Print the function name explicitly if we don't have a debug location (which
   // makes the diagnostic less useful) or if we're going to emit a raw error.
   if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())
     R << (" (in function: " + MF.getName() + ")").str();
 
   if (TPC.isGlobalISelAbortEnabled())
     report_fatal_error(Twine(R.getMsg()));
   else
     ORE.emit(R);
 }
 
 IRTranslator::IRTranslator(CodeGenOpt::Level optlevel)
     : MachineFunctionPass(ID), OptLevel(optlevel) {}
 
 #ifndef NDEBUG
 namespace {
 /// Verify that every instruction created has the same DILocation as the
 /// instruction being translated.
 class DILocationVerifier : public GISelChangeObserver {
   const Instruction *CurrInst = nullptr;
 
 public:
   DILocationVerifier() = default;
   ~DILocationVerifier() = default;
 
   const Instruction *getCurrentInst() const { return CurrInst; }
   void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
 
   void erasingInstr(MachineInstr &MI) override {}
   void changingInstr(MachineInstr &MI) override {}
   void changedInstr(MachineInstr &MI) override {}
 
   void createdInstr(MachineInstr &MI) override {
     assert(getCurrentInst() && "Inserted instruction without a current MI");
 
     // Only print the check message if we're actually checking it.
 #ifndef NDEBUG
     LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
                       << " was copied to " << MI);
 #endif
     // We allow insts in the entry block to have a debug loc line of 0 because
     // they could have originated from constants, and we don't want a jumpy
     // debug experience.
     assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
             MI.getDebugLoc().getLine() == 0) &&
            "Line info was not transferred to all instructions");
   }
 };
 } // namespace
 #endif // ifndef NDEBUG
 
 
 void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<StackProtector>();
   AU.addRequired<TargetPassConfig>();
   AU.addRequired<GISelCSEAnalysisWrapperPass>();
   if (OptLevel != CodeGenOpt::None)
     AU.addRequired<BranchProbabilityInfoWrapperPass>();
   AU.addRequired<TargetLibraryInfoWrapperPass>();
   AU.addPreserved<TargetLibraryInfoWrapperPass>();
   getSelectionDAGFallbackAnalysisUsage(AU);
   MachineFunctionPass::getAnalysisUsage(AU);
 }
 
 IRTranslator::ValueToVRegInfo::VRegListT &
 IRTranslator::allocateVRegs(const Value &Val) {
   auto VRegsIt = VMap.findVRegs(Val);
   if (VRegsIt != VMap.vregs_end())
     return *VRegsIt->second;
   auto *Regs = VMap.getVRegs(Val);
   auto *Offsets = VMap.getOffsets(Val);
   SmallVector<LLT, 4> SplitTys;
   computeValueLLTs(*DL, *Val.getType(), SplitTys,
                    Offsets->empty() ? Offsets : nullptr);
   for (unsigned i = 0; i < SplitTys.size(); ++i)
     Regs->push_back(0);
   return *Regs;
 }
 
 ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
   auto VRegsIt = VMap.findVRegs(Val);
   if (VRegsIt != VMap.vregs_end())
     return *VRegsIt->second;
 
   if (Val.getType()->isVoidTy())
     return *VMap.getVRegs(Val);
 
   // Create entry for this type.
   auto *VRegs = VMap.getVRegs(Val);
   auto *Offsets = VMap.getOffsets(Val);
 
   assert(Val.getType()->isSized() &&
          "Don't know how to create an empty vreg");
 
   SmallVector<LLT, 4> SplitTys;
   computeValueLLTs(*DL, *Val.getType(), SplitTys,
                    Offsets->empty() ? Offsets : nullptr);
 
   if (!isa<Constant>(Val)) {
     for (auto Ty : SplitTys)
       VRegs->push_back(MRI->createGenericVirtualRegister(Ty));
     return *VRegs;
   }
 
   if (Val.getType()->isAggregateType()) {
     // UndefValue, ConstantAggregateZero
     auto &C = cast<Constant>(Val);
     unsigned Idx = 0;
     while (auto Elt = C.getAggregateElement(Idx++)) {
       auto EltRegs = getOrCreateVRegs(*Elt);
       llvm::copy(EltRegs, std::back_inserter(*VRegs));
     }
   } else {
     assert(SplitTys.size() == 1 && "unexpectedly split LLT");
     VRegs->push_back(MRI->createGenericVirtualRegister(SplitTys[0]));
     bool Success = translate(cast<Constant>(Val), VRegs->front());
     if (!Success) {
       OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
                                  MF->getFunction().getSubprogram(),
                                  &MF->getFunction().getEntryBlock());
       R << "unable to translate constant: " << ore::NV("Type", Val.getType());
       reportTranslationError(*MF, *TPC, *ORE, R);
       return *VRegs;
     }
   }
 
   return *VRegs;
 }
 
 int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
   auto MapEntry = FrameIndices.find(&AI);
   if (MapEntry != FrameIndices.end())
     return MapEntry->second;
 
   uint64_t ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
   uint64_t Size =
       ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();
 
   // Always allocate at least one byte.
   Size = std::max<uint64_t>(Size, 1u);
 
   int &FI = FrameIndices[&AI];
   FI = MF->getFrameInfo().CreateStackObject(Size, AI.getAlign(), false, &AI);
   return FI;
 }
 
 Align IRTranslator::getMemOpAlign(const Instruction &I) {
   if (const StoreInst *SI = dyn_cast<StoreInst>(&I))
     return SI->getAlign();
   if (const LoadInst *LI = dyn_cast<LoadInst>(&I))
     return LI->getAlign();
   if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I))
     return AI->getAlign();
   if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I))
     return AI->getAlign();
 
   OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
   R << "unable to translate memop: " << ore::NV("Opcode", &I);
   reportTranslationError(*MF, *TPC, *ORE, R);
   return Align(1);
 }
 
 MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
   MachineBasicBlock *&MBB = BBToMBB[&BB];
   assert(MBB && "BasicBlock was not encountered before");
   return *MBB;
 }
 
 void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
   assert(NewPred && "new predecessor must be a real MachineBasicBlock");
   MachinePreds[Edge].push_back(NewPred);
 }
 
 bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
                                      MachineIRBuilder &MIRBuilder) {
   // Get or create a virtual register for each value.
   // Unless the value is a Constant => loadimm cst?
   // or inline constant each time?
   // Creation of a virtual register needs to have a size.
   Register Op0 = getOrCreateVReg(*U.getOperand(0));
   Register Op1 = getOrCreateVReg(*U.getOperand(1));
   Register Res = getOrCreateVReg(U);
   uint16_t Flags = 0;
   if (isa<Instruction>(U)) {
     const Instruction &I = cast<Instruction>(U);
     Flags = MachineInstr::copyFlagsFromInstruction(I);
   }
 
   MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);
   return true;
 }
 
 bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
                                     MachineIRBuilder &MIRBuilder) {
   Register Op0 = getOrCreateVReg(*U.getOperand(0));
   Register Res = getOrCreateVReg(U);
   uint16_t Flags = 0;
   if (isa<Instruction>(U)) {
     const Instruction &I = cast<Instruction>(U);
     Flags = MachineInstr::copyFlagsFromInstruction(I);
   }
   MIRBuilder.buildInstr(Opcode, {Res}, {Op0}, Flags);
   return true;
 }
 
 bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
   return translateUnaryOp(TargetOpcode::G_FNEG, U, MIRBuilder);
 }
 
 bool IRTranslator::translateCompare(const User &U,
                                     MachineIRBuilder &MIRBuilder) {
   auto *CI = dyn_cast<CmpInst>(&U);
   Register Op0 = getOrCreateVReg(*U.getOperand(0));
   Register Op1 = getOrCreateVReg(*U.getOperand(1));
   Register Res = getOrCreateVReg(U);
   CmpInst::Predicate Pred =
       CI ? CI->getPredicate() : static_cast<CmpInst::Predicate>(
                                     cast<ConstantExpr>(U).getPredicate());
   if (CmpInst::isIntPredicate(Pred))
     MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
   else if (Pred == CmpInst::FCMP_FALSE)
     MIRBuilder.buildCopy(
         Res, getOrCreateVReg(*Constant::getNullValue(U.getType())));
   else if (Pred == CmpInst::FCMP_TRUE)
     MIRBuilder.buildCopy(
         Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
   else {
     uint16_t Flags = 0;
     if (CI)
       Flags = MachineInstr::copyFlagsFromInstruction(*CI);
     MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
   }
 
   return true;
 }
 
 bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
   const ReturnInst &RI = cast<ReturnInst>(U);
   const Value *Ret = RI.getReturnValue();
   if (Ret && DL->getTypeStoreSize(Ret->getType()) == 0)
     Ret = nullptr;
 
   ArrayRef<Register> VRegs;
   if (Ret)
     VRegs = getOrCreateVRegs(*Ret);
 
   Register SwiftErrorVReg = 0;
   if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {
     SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
         &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
   }
 
   // The target may mess up with the insertion point, but
   // this is not important as a return is the last instruction
   // of the block anyway.
   return CLI->lowerReturn(MIRBuilder, Ret, VRegs, FuncInfo, SwiftErrorVReg);
 }
 
 void IRTranslator::emitBranchForMergedCondition(
     const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
     MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
     BranchProbability TProb, BranchProbability FProb, bool InvertCond) {
   // If the leaf of the tree is a comparison, merge the condition into
   // the caseblock.
   if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
     CmpInst::Predicate Condition;
     if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
       Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();
     } else {
       const FCmpInst *FC = cast<FCmpInst>(Cond);
       Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();
     }
 
     SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(0),
                            BOp->getOperand(1), nullptr, TBB, FBB, CurBB,
                            CurBuilder->getDebugLoc(), TProb, FProb);
     SL->SwitchCases.push_back(CB);
     return;
   }
 
   // Create a CaseBlock record representing this branch.
   CmpInst::Predicate Pred = InvertCond ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ;
   SwitchCG::CaseBlock CB(
       Pred, false, Cond, ConstantInt::getTrue(MF->getFunction().getContext()),
       nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);
   SL->SwitchCases.push_back(CB);
 }
 
 static bool isValInBlock(const Value *V, const BasicBlock *BB) {
   if (const Instruction *I = dyn_cast<Instruction>(V))
     return I->getParent() == BB;
   return true;
 }
 
 void IRTranslator::findMergedConditions(
     const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
     MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,
     Instruction::BinaryOps Opc, BranchProbability TProb,
     BranchProbability FProb, bool InvertCond) {
   using namespace PatternMatch;
   assert((Opc == Instruction::And || Opc == Instruction::Or) &&
          "Expected Opc to be AND/OR");
   // Skip over not part of the tree and remember to invert op and operands at
   // next level.
   Value *NotCond;
   if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
       isValInBlock(NotCond, CurBB->getBasicBlock())) {
     findMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
                          !InvertCond);
     return;
   }
 
   const Instruction *BOp = dyn_cast<Instruction>(Cond);
   const Value *BOpOp0, *BOpOp1;
   // Compute the effective opcode for Cond, taking into account whether it needs
   // to be inverted, e.g.
   //   and (not (or A, B)), C
   // gets lowered as
   //   and (and (not A, not B), C)
   Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
   if (BOp) {
     BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
                ? Instruction::And
                : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
                       ? Instruction::Or
                       : (Instruction::BinaryOps)0);
     if (InvertCond) {
       if (BOpc == Instruction::And)
         BOpc = Instruction::Or;
       else if (BOpc == Instruction::Or)
         BOpc = Instruction::And;
     }
   }
 
   // If this node is not part of the or/and tree, emit it as a branch.
   // Note that all nodes in the tree should have same opcode.
   bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
   if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
       !isValInBlock(BOpOp0, CurBB->getBasicBlock()) ||
       !isValInBlock(BOpOp1, CurBB->getBasicBlock())) {
     emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,
                                  InvertCond);
     return;
   }
 
   //  Create TmpBB after CurBB.
   MachineFunction::iterator BBI(CurBB);
   MachineBasicBlock *TmpBB =
       MF->CreateMachineBasicBlock(CurBB->getBasicBlock());
   CurBB->getParent()->insert(++BBI, TmpBB);
 
   if (Opc == Instruction::Or) {
     // Codegen X | Y as:
     // BB1:
     //   jmp_if_X TBB
     //   jmp TmpBB
     // TmpBB:
     //   jmp_if_Y TBB
     //   jmp FBB
     //
 
     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
     // The requirement is that
     //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
     //     = TrueProb for original BB.
     // Assuming the original probabilities are A and B, one choice is to set
     // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
     // A/(1+B) and 2B/(1+B). This choice assumes that
     //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
     // Another choice is to assume TrueProb for BB1 equals to TrueProb for
     // TmpBB, but the math is more complicated.
 
     auto NewTrueProb = TProb / 2;
     auto NewFalseProb = TProb / 2 + FProb;
     // Emit the LHS condition.
     findMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
                          NewFalseProb, InvertCond);
 
     // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
     SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
     // Emit the RHS condition into TmpBB.
     findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
                          Probs[1], InvertCond);
   } else {
     assert(Opc == Instruction::And && "Unknown merge op!");
     // Codegen X & Y as:
     // BB1:
     //   jmp_if_X TmpBB
     //   jmp FBB
     // TmpBB:
     //   jmp_if_Y TBB
     //   jmp FBB
     //
     //  This requires creation of TmpBB after CurBB.
 
     // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
     // The requirement is that
     //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
     //     = FalseProb for original BB.
     // Assuming the original probabilities are A and B, one choice is to set
     // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
     // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
     // TrueProb for BB1 * FalseProb for TmpBB.
 
     auto NewTrueProb = TProb + FProb / 2;
     auto NewFalseProb = FProb / 2;
     // Emit the LHS condition.
     findMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
                          NewFalseProb, InvertCond);
 
     // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
     SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
     BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
     // Emit the RHS condition into TmpBB.
     findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
                          Probs[1], InvertCond);
   }
 }
 
 bool IRTranslator::shouldEmitAsBranches(
     const std::vector<SwitchCG::CaseBlock> &Cases) {
   // For multiple cases, it's better to emit as branches.
   if (Cases.size() != 2)
     return true;
 
   // If this is two comparisons of the same values or'd or and'd together, they
   // will get folded into a single comparison, so don't emit two blocks.
   if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
        Cases[0].CmpRHS == Cases[1].CmpRHS) ||
       (Cases[0].CmpRHS == Cases[1].CmpLHS &&
        Cases[0].CmpLHS == Cases[1].CmpRHS)) {
     return false;
   }
 
   // Handle: (X != null) | (Y != null) --> (X|Y) != 0
   // Handle: (X == null) & (Y == null) --> (X|Y) == 0
   if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
       Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&
       isa<Constant>(Cases[0].CmpRHS) &&
       cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
     if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&
         Cases[0].TrueBB == Cases[1].ThisBB)
       return false;
     if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&
         Cases[0].FalseBB == Cases[1].ThisBB)
       return false;
   }
 
   return true;
 }
 
 bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
   const BranchInst &BrInst = cast<BranchInst>(U);
   auto &CurMBB = MIRBuilder.getMBB();
   auto *Succ0MBB = &getMBB(*BrInst.getSuccessor(0));
 
   if (BrInst.isUnconditional()) {
     // If the unconditional target is the layout successor, fallthrough.
     if (OptLevel == CodeGenOpt::None || !CurMBB.isLayoutSuccessor(Succ0MBB))
       MIRBuilder.buildBr(*Succ0MBB);
 
     // Link successors.
     for (const BasicBlock *Succ : successors(&BrInst))
       CurMBB.addSuccessor(&getMBB(*Succ));
     return true;
   }
 
   // If this condition is one of the special cases we handle, do special stuff
   // now.
   const Value *CondVal = BrInst.getCondition();
   MachineBasicBlock *Succ1MBB = &getMBB(*BrInst.getSuccessor(1));
 
   const auto &TLI = *MF->getSubtarget().getTargetLowering();
 
   // If this is a series of conditions that are or'd or and'd together, emit
   // this as a sequence of branches instead of setcc's with and/or operations.
   // As long as jumps are not expensive (exceptions for multi-use logic ops,
   // unpredictable branches, and vector extracts because those jumps are likely
   // expensive for any target), this should improve performance.
   // For example, instead of something like:
   //     cmp A, B
   //     C = seteq
   //     cmp D, E
   //     F = setle
   //     or C, F
   //     jnz foo
   // Emit:
   //     cmp A, B
   //     je foo
   //     cmp D, E
   //     jle foo
   using namespace PatternMatch;
   const Instruction *CondI = dyn_cast<Instruction>(CondVal);
   if (!TLI.isJumpExpensive() && CondI && CondI->hasOneUse() &&
       !BrInst.hasMetadata(LLVMContext::MD_unpredictable)) {
     Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
     Value *Vec;
     const Value *BOp0, *BOp1;
     if (match(CondI, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
       Opcode = Instruction::And;
     else if (match(CondI, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
       Opcode = Instruction::Or;
 
     if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
                     match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
       findMergedConditions(CondI, Succ0MBB, Succ1MBB, &CurMBB, &CurMBB, Opcode,
                            getEdgeProbability(&CurMBB, Succ0MBB),
                            getEdgeProbability(&CurMBB, Succ1MBB),
                            /*InvertCond=*/false);
       assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");
 
       // Allow some cases to be rejected.
       if (shouldEmitAsBranches(SL->SwitchCases)) {
         // Emit the branch for this block.
         emitSwitchCase(SL->SwitchCases[0], &CurMBB, *CurBuilder);
         SL->SwitchCases.erase(SL->SwitchCases.begin());
         return true;
       }
 
       // Okay, we decided not to do this, remove any inserted MBB's and clear
       // SwitchCases.
       for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)
         MF->erase(SL->SwitchCases[I].ThisBB);
 
       SL->SwitchCases.clear();
     }
   }
 
   // Create a CaseBlock record representing this branch.
   SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,
                          ConstantInt::getTrue(MF->getFunction().getContext()),
                          nullptr, Succ0MBB, Succ1MBB, &CurMBB,
                          CurBuilder->getDebugLoc());
 
   // Use emitSwitchCase to actually insert the fast branch sequence for this
   // cond branch.
   emitSwitchCase(CB, &CurMBB, *CurBuilder);
   return true;
 }
 
 void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
                                         MachineBasicBlock *Dst,
                                         BranchProbability Prob) {
   if (!FuncInfo.BPI) {
     Src->addSuccessorWithoutProb(Dst);
     return;
   }
   if (Prob.isUnknown())
     Prob = getEdgeProbability(Src, Dst);
   Src->addSuccessor(Dst, Prob);
 }
 
 BranchProbability
 IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
                                  const MachineBasicBlock *Dst) const {
   const BasicBlock *SrcBB = Src->getBasicBlock();
   const BasicBlock *DstBB = Dst->getBasicBlock();
   if (!FuncInfo.BPI) {
     // If BPI is not available, set the default probability as 1 / N, where N is
     // the number of successors.
     auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
     return BranchProbability(1, SuccSize);
   }
   return FuncInfo.BPI->getEdgeProbability(SrcBB, DstBB);
 }
 
 bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
   using namespace SwitchCG;
   // Extract cases from the switch.
   const SwitchInst &SI = cast<SwitchInst>(U);
   BranchProbabilityInfo *BPI = FuncInfo.BPI;
   CaseClusterVector Clusters;
   Clusters.reserve(SI.getNumCases());
   for (auto &I : SI.cases()) {
     MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor());
     assert(Succ && "Could not find successor mbb in mapping");
     const ConstantInt *CaseVal = I.getCaseValue();
     BranchProbability Prob =
         BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
             : BranchProbability(1, SI.getNumCases() + 1);
     Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
   }
 
   MachineBasicBlock *DefaultMBB = &getMBB(*SI.getDefaultDest());
 
   // Cluster adjacent cases with the same destination. We do this at all
   // optimization levels because it's cheap to do and will make codegen faster
   // if there are many clusters.
   sortAndRangeify(Clusters);
 
   MachineBasicBlock *SwitchMBB = &getMBB(*SI.getParent());
 
   // If there is only the default destination, jump there directly.
   if (Clusters.empty()) {
     SwitchMBB->addSuccessor(DefaultMBB);
     if (DefaultMBB != SwitchMBB->getNextNode())
       MIB.buildBr(*DefaultMBB);
     return true;
   }
 
   SL->findJumpTables(Clusters, &SI, DefaultMBB, nullptr, nullptr);
   SL->findBitTestClusters(Clusters, &SI);
 
   LLVM_DEBUG({
     dbgs() << "Case clusters: ";
     for (const CaseCluster &C : Clusters) {
       if (C.Kind == CC_JumpTable)
         dbgs() << "JT:";
       if (C.Kind == CC_BitTests)
         dbgs() << "BT:";
 
       C.Low->getValue().print(dbgs(), true);
       if (C.Low != C.High) {
         dbgs() << '-';
         C.High->getValue().print(dbgs(), true);
       }
       dbgs() << ' ';
     }
     dbgs() << '\n';
   });
 
   assert(!Clusters.empty());
   SwitchWorkList WorkList;
   CaseClusterIt First = Clusters.begin();
   CaseClusterIt Last = Clusters.end() - 1;
   auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
   WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
 
   // FIXME: At the moment we don't do any splitting optimizations here like
   // SelectionDAG does, so this worklist only has one entry.
   while (!WorkList.empty()) {
     SwitchWorkListItem W = WorkList.pop_back_val();
     if (!lowerSwitchWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
       return false;
   }
   return true;
 }
 
 void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
                                  MachineBasicBlock *MBB) {
   // Emit the code for the jump table
   assert(JT.Reg != -1U && "Should lower JT Header first!");
   MachineIRBuilder MIB(*MBB->getParent());
   MIB.setMBB(*MBB);
   MIB.setDebugLoc(CurBuilder->getDebugLoc());
 
   Type *PtrIRTy = Type::getInt8PtrTy(MF->getFunction().getContext());
   const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
 
   auto Table = MIB.buildJumpTable(PtrTy, JT.JTI);
   MIB.buildBrJT(Table.getReg(0), JT.JTI, JT.Reg);
 }
 
 bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
                                        SwitchCG::JumpTableHeader &JTH,
                                        MachineBasicBlock *HeaderBB) {
   MachineIRBuilder MIB(*HeaderBB->getParent());
   MIB.setMBB(*HeaderBB);
   MIB.setDebugLoc(CurBuilder->getDebugLoc());
 
   const Value &SValue = *JTH.SValue;
   // Subtract the lowest switch case value from the value being switched on.
   const LLT SwitchTy = getLLTForType(*SValue.getType(), *DL);
   Register SwitchOpReg = getOrCreateVReg(SValue);
   auto FirstCst = MIB.buildConstant(SwitchTy, JTH.First);
   auto Sub = MIB.buildSub({SwitchTy}, SwitchOpReg, FirstCst);
 
   // This value may be smaller or larger than the target's pointer type, and
   // therefore require extension or truncating.
   Type *PtrIRTy = SValue.getType()->getPointerTo();
   const LLT PtrScalarTy = LLT::scalar(DL->getTypeSizeInBits(PtrIRTy));
   Sub = MIB.buildZExtOrTrunc(PtrScalarTy, Sub);
 
   JT.Reg = Sub.getReg(0);
 
   if (JTH.FallthroughUnreachable) {
     if (JT.MBB != HeaderBB->getNextNode())
       MIB.buildBr(*JT.MBB);
     return true;
   }
 
   // Emit the range check for the jump table, and branch to the default block
   // for the switch statement if the value being switched on exceeds the
   // largest case in the switch.
   auto Cst = getOrCreateVReg(
       *ConstantInt::get(SValue.getType(), JTH.Last - JTH.First));
   Cst = MIB.buildZExtOrTrunc(PtrScalarTy, Cst).getReg(0);
   auto Cmp = MIB.buildICmp(CmpInst::ICMP_UGT, LLT::scalar(1), Sub, Cst);
 
   auto BrCond = MIB.buildBrCond(Cmp.getReg(0), *JT.Default);
 
   // Avoid emitting unnecessary branches to the next block.
   if (JT.MBB != HeaderBB->getNextNode())
     BrCond = MIB.buildBr(*JT.MBB);
   return true;
 }
 
 void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
                                   MachineBasicBlock *SwitchBB,
                                   MachineIRBuilder &MIB) {
   Register CondLHS = getOrCreateVReg(*CB.CmpLHS);
   Register Cond;
   DebugLoc OldDbgLoc = MIB.getDebugLoc();
   MIB.setDebugLoc(CB.DbgLoc);
   MIB.setMBB(*CB.ThisBB);
 
   if (CB.PredInfo.NoCmp) {
     // Branch or fall through to TrueBB.
     addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
     addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
                       CB.ThisBB);
     CB.ThisBB->normalizeSuccProbs();
     if (CB.TrueBB != CB.ThisBB->getNextNode())
       MIB.buildBr(*CB.TrueBB);
     MIB.setDebugLoc(OldDbgLoc);
     return;
   }
 
   const LLT i1Ty = LLT::scalar(1);
   // Build the compare.
   if (!CB.CmpMHS) {
     const auto *CI = dyn_cast<ConstantInt>(CB.CmpRHS);
     // For conditional branch lowering, we might try to do something silly like
     // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,
     // just re-use the existing condition vreg.
     if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI &&
         CI->getZExtValue() == 1 && CB.PredInfo.Pred == CmpInst::ICMP_EQ) {
       Cond = CondLHS;
     } else {
       Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
       if (CmpInst::isFPPredicate(CB.PredInfo.Pred))
         Cond =
             MIB.buildFCmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
       else
         Cond =
             MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
     }
   } else {
     assert(CB.PredInfo.Pred == CmpInst::ICMP_SLE &&
            "Can only handle SLE ranges");
 
     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
     const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
 
     Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS);
     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
       Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
       Cond =
           MIB.buildICmp(CmpInst::ICMP_SLE, i1Ty, CmpOpReg, CondRHS).getReg(0);
     } else {
       const LLT CmpTy = MRI->getType(CmpOpReg);
       auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS);
       auto Diff = MIB.buildConstant(CmpTy, High - Low);
       Cond = MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, Sub, Diff).getReg(0);
     }
   }
 
   // Update successor info
   addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
 
   addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
                     CB.ThisBB);
 
   // TrueBB and FalseBB are always different unless the incoming IR is
   // degenerate. This only happens when running llc on weird IR.
   if (CB.TrueBB != CB.FalseBB)
     addSuccessorWithProb(CB.ThisBB, CB.FalseBB, CB.FalseProb);
   CB.ThisBB->normalizeSuccProbs();
 
   addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
                     CB.ThisBB);
 
   MIB.buildBrCond(Cond, *CB.TrueBB);
   MIB.buildBr(*CB.FalseBB);
   MIB.setDebugLoc(OldDbgLoc);
 }
 
 bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
                                           MachineBasicBlock *SwitchMBB,
                                           MachineBasicBlock *CurMBB,
                                           MachineBasicBlock *DefaultMBB,
                                           MachineIRBuilder &MIB,
                                           MachineFunction::iterator BBI,
                                           BranchProbability UnhandledProbs,
                                           SwitchCG::CaseClusterIt I,
                                           MachineBasicBlock *Fallthrough,
                                           bool FallthroughUnreachable) {
   using namespace SwitchCG;
   MachineFunction *CurMF = SwitchMBB->getParent();
   // FIXME: Optimize away range check based on pivot comparisons.
   JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
   SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
   BranchProbability DefaultProb = W.DefaultProb;
 
   // The jump block hasn't been inserted yet; insert it here.
   MachineBasicBlock *JumpMBB = JT->MBB;
   CurMF->insert(BBI, JumpMBB);
 
   // Since the jump table block is separate from the switch block, we need
   // to keep track of it as a machine predecessor to the default block,
   // otherwise we lose the phi edges.
   addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
                     CurMBB);
   addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
                     JumpMBB);
 
   auto JumpProb = I->Prob;
   auto FallthroughProb = UnhandledProbs;
 
   // If the default statement is a target of the jump table, we evenly
   // distribute the default probability to successors of CurMBB. Also
   // update the probability on the edge from JumpMBB to Fallthrough.
   for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
                                         SE = JumpMBB->succ_end();
        SI != SE; ++SI) {
     if (*SI == DefaultMBB) {
       JumpProb += DefaultProb / 2;
       FallthroughProb -= DefaultProb / 2;
       JumpMBB->setSuccProbability(SI, DefaultProb / 2);
       JumpMBB->normalizeSuccProbs();
     } else {
       // Also record edges from the jump table block to it's successors.
       addMachineCFGPred({SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
                         JumpMBB);
     }
   }
 
   if (FallthroughUnreachable)
     JTH->FallthroughUnreachable = true;
 
   if (!JTH->FallthroughUnreachable)
     addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
   addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
   CurMBB->normalizeSuccProbs();
 
   // The jump table header will be inserted in our current block, do the
   // range check, and fall through to our fallthrough block.
   JTH->HeaderBB = CurMBB;
   JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
 
   // If we're in the right place, emit the jump table header right now.
   if (CurMBB == SwitchMBB) {
     if (!emitJumpTableHeader(*JT, *JTH, CurMBB))
       return false;
     JTH->Emitted = true;
   }
   return true;
 }
 bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
                                             Value *Cond,
                                             MachineBasicBlock *Fallthrough,
                                             bool FallthroughUnreachable,
                                             BranchProbability UnhandledProbs,
                                             MachineBasicBlock *CurMBB,
                                             MachineIRBuilder &MIB,
                                             MachineBasicBlock *SwitchMBB) {
   using namespace SwitchCG;
   const Value *RHS, *LHS, *MHS;
   CmpInst::Predicate Pred;
   if (I->Low == I->High) {
     // Check Cond == I->Low.
     Pred = CmpInst::ICMP_EQ;
     LHS = Cond;
     RHS = I->Low;
     MHS = nullptr;
   } else {
     // Check I->Low <= Cond <= I->High.
     Pred = CmpInst::ICMP_SLE;
     LHS = I->Low;
     MHS = Cond;
     RHS = I->High;
   }
 
   // If Fallthrough is unreachable, fold away the comparison.
   // The false probability is the sum of all unhandled cases.
   CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
                CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
 
   emitSwitchCase(CB, SwitchMBB, MIB);
   return true;
 }
 
 void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B,
                                      MachineBasicBlock *SwitchBB) {
   MachineIRBuilder &MIB = *CurBuilder;
   MIB.setMBB(*SwitchBB);
 
   // Subtract the minimum value.
   Register SwitchOpReg = getOrCreateVReg(*B.SValue);
 
   LLT SwitchOpTy = MRI->getType(SwitchOpReg);
   Register MinValReg = MIB.buildConstant(SwitchOpTy, B.First).getReg(0);
   auto RangeSub = MIB.buildSub(SwitchOpTy, SwitchOpReg, MinValReg);
 
   Type *PtrIRTy = Type::getInt8PtrTy(MF->getFunction().getContext());
   const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
 
   LLT MaskTy = SwitchOpTy;
   if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||
       !isPowerOf2_32(MaskTy.getSizeInBits()))
     MaskTy = LLT::scalar(PtrTy.getSizeInBits());
   else {
     // Ensure that the type will fit the mask value.
     for (unsigned I = 0, E = B.Cases.size(); I != E; ++I) {
       if (!isUIntN(SwitchOpTy.getSizeInBits(), B.Cases[I].Mask)) {
         // Switch table case range are encoded into series of masks.
         // Just use pointer type, it's guaranteed to fit.
         MaskTy = LLT::scalar(PtrTy.getSizeInBits());
         break;
       }
     }
   }
   Register SubReg = RangeSub.getReg(0);
   if (SwitchOpTy != MaskTy)
     SubReg = MIB.buildZExtOrTrunc(MaskTy, SubReg).getReg(0);
 
   B.RegVT = getMVTForLLT(MaskTy);
   B.Reg = SubReg;
 
   MachineBasicBlock *MBB = B.Cases[0].ThisBB;
 
   if (!B.FallthroughUnreachable)
     addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
   addSuccessorWithProb(SwitchBB, MBB, B.Prob);
 
   SwitchBB->normalizeSuccProbs();
 
   if (!B.FallthroughUnreachable) {
     // Conditional branch to the default block.
     auto RangeCst = MIB.buildConstant(SwitchOpTy, B.Range);
     auto RangeCmp = MIB.buildICmp(CmpInst::Predicate::ICMP_UGT, LLT::scalar(1),
                                   RangeSub, RangeCst);
     MIB.buildBrCond(RangeCmp, *B.Default);
   }
 
   // Avoid emitting unnecessary branches to the next block.
   if (MBB != SwitchBB->getNextNode())
     MIB.buildBr(*MBB);
 }
 
 void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB,
                                    MachineBasicBlock *NextMBB,
                                    BranchProbability BranchProbToNext,
                                    Register Reg, SwitchCG::BitTestCase &B,
                                    MachineBasicBlock *SwitchBB) {
   MachineIRBuilder &MIB = *CurBuilder;
   MIB.setMBB(*SwitchBB);
 
   LLT SwitchTy = getLLTForMVT(BB.RegVT);
   Register Cmp;
   unsigned PopCount = countPopulation(B.Mask);
   if (PopCount == 1) {
     // Testing for a single bit; just compare the shift count with what it
     // would need to be to shift a 1 bit in that position.
     auto MaskTrailingZeros =
         MIB.buildConstant(SwitchTy, countTrailingZeros(B.Mask));
     Cmp =
         MIB.buildICmp(ICmpInst::ICMP_EQ, LLT::scalar(1), Reg, MaskTrailingZeros)
             .getReg(0);
   } else if (PopCount == BB.Range) {
     // There is only one zero bit in the range, test for it directly.
     auto MaskTrailingOnes =
         MIB.buildConstant(SwitchTy, countTrailingOnes(B.Mask));
     Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Reg, MaskTrailingOnes)
               .getReg(0);
   } else {
     // Make desired shift.
     auto CstOne = MIB.buildConstant(SwitchTy, 1);
     auto SwitchVal = MIB.buildShl(SwitchTy, CstOne, Reg);
 
     // Emit bit tests and jumps.
     auto CstMask = MIB.buildConstant(SwitchTy, B.Mask);
     auto AndOp = MIB.buildAnd(SwitchTy, SwitchVal, CstMask);
     auto CstZero = MIB.buildConstant(SwitchTy, 0);
     Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), AndOp, CstZero)
               .getReg(0);
   }
 
   // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
   addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
   // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
   addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
   // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
   // one as they are relative probabilities (and thus work more like weights),
   // and hence we need to normalize them to let the sum of them become one.
   SwitchBB->normalizeSuccProbs();
 
   // Record the fact that the IR edge from the header to the bit test target
   // will go through our new block. Neeeded for PHIs to have nodes added.
   addMachineCFGPred({BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},
                     SwitchBB);
 
   MIB.buildBrCond(Cmp, *B.TargetBB);
 
   // Avoid emitting unnecessary branches to the next block.
   if (NextMBB != SwitchBB->getNextNode())
     MIB.buildBr(*NextMBB);
 }
 
 bool IRTranslator::lowerBitTestWorkItem(
     SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
     MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
     MachineIRBuilder &MIB, MachineFunction::iterator BBI,
     BranchProbability DefaultProb, BranchProbability UnhandledProbs,
     SwitchCG::CaseClusterIt I, MachineBasicBlock *Fallthrough,
     bool FallthroughUnreachable) {
   using namespace SwitchCG;
   MachineFunction *CurMF = SwitchMBB->getParent();
   // FIXME: Optimize away range check based on pivot comparisons.
   BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
   // The bit test blocks haven't been inserted yet; insert them here.
   for (BitTestCase &BTC : BTB->Cases)
     CurMF->insert(BBI, BTC.ThisBB);
 
   // Fill in fields of the BitTestBlock.
   BTB->Parent = CurMBB;
   BTB->Default = Fallthrough;
 
   BTB->DefaultProb = UnhandledProbs;
   // If the cases in bit test don't form a contiguous range, we evenly
   // distribute the probability on the edge to Fallthrough to two
   // successors of CurMBB.
   if (!BTB->ContiguousRange) {
     BTB->Prob += DefaultProb / 2;
     BTB->DefaultProb -= DefaultProb / 2;
   }
 
   if (FallthroughUnreachable)
     BTB->FallthroughUnreachable = true;
 
   // If we're in the right place, emit the bit test header right now.
   if (CurMBB == SwitchMBB) {
     emitBitTestHeader(*BTB, SwitchMBB);
     BTB->Emitted = true;
   }
   return true;
 }
 
 bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
                                        Value *Cond,
                                        MachineBasicBlock *SwitchMBB,
                                        MachineBasicBlock *DefaultMBB,
                                        MachineIRBuilder &MIB) {
   using namespace SwitchCG;
   MachineFunction *CurMF = FuncInfo.MF;
   MachineBasicBlock *NextMBB = nullptr;
   MachineFunction::iterator BBI(W.MBB);
   if (++BBI != FuncInfo.MF->end())
     NextMBB = &*BBI;
 
   if (EnableOpts) {
     // Here, we order cases by probability so the most likely case will be
     // checked first. However, two clusters can have the same probability in
     // which case their relative ordering is non-deterministic. So we use Low
     // as a tie-breaker as clusters are guaranteed to never overlap.
     llvm::sort(W.FirstCluster, W.LastCluster + 1,
                [](const CaseCluster &a, const CaseCluster &b) {
                  return a.Prob != b.Prob
                             ? a.Prob > b.Prob
                             : a.Low->getValue().slt(b.Low->getValue());
                });
 
     // Rearrange the case blocks so that the last one falls through if possible
     // without changing the order of probabilities.
     for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
       --I;
       if (I->Prob > W.LastCluster->Prob)
         break;
       if (I->Kind == CC_Range && I->MBB == NextMBB) {
         std::swap(*I, *W.LastCluster);
         break;
       }
     }
   }
 
   // Compute total probability.
   BranchProbability DefaultProb = W.DefaultProb;
   BranchProbability UnhandledProbs = DefaultProb;
   for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
     UnhandledProbs += I->Prob;
 
   MachineBasicBlock *CurMBB = W.MBB;
   for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
     bool FallthroughUnreachable = false;
     MachineBasicBlock *Fallthrough;
     if (I == W.LastCluster) {
       // For the last cluster, fall through to the default destination.
       Fallthrough = DefaultMBB;
       FallthroughUnreachable = isa<UnreachableInst>(
           DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
     } else {
       Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
       CurMF->insert(BBI, Fallthrough);
     }
     UnhandledProbs -= I->Prob;
 
     switch (I->Kind) {
     case CC_BitTests: {
       if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
                                 DefaultProb, UnhandledProbs, I, Fallthrough,
                                 FallthroughUnreachable)) {
         LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");
         return false;
       }
       break;
     }
 
     case CC_JumpTable: {
       if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
                                   UnhandledProbs, I, Fallthrough,
                                   FallthroughUnreachable)) {
         LLVM_DEBUG(dbgs() << "Failed to lower jump table");
         return false;
       }
       break;
     }
     case CC_Range: {
       if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
                                     FallthroughUnreachable, UnhandledProbs,
                                     CurMBB, MIB, SwitchMBB)) {
         LLVM_DEBUG(dbgs() << "Failed to lower switch range");
         return false;
       }
       break;
     }
     }
     CurMBB = Fallthrough;
   }
 
   return true;
 }
 
 bool IRTranslator::translateIndirectBr(const User &U,
                                        MachineIRBuilder &MIRBuilder) {
   const IndirectBrInst &BrInst = cast<IndirectBrInst>(U);
 
   const Register Tgt = getOrCreateVReg(*BrInst.getAddress());
   MIRBuilder.buildBrIndirect(Tgt);
 
   // Link successors.
   SmallPtrSet<const BasicBlock *, 32> AddedSuccessors;
   MachineBasicBlock &CurBB = MIRBuilder.getMBB();
   for (const BasicBlock *Succ : successors(&BrInst)) {
     // It's legal for indirectbr instructions to have duplicate blocks in the
     // destination list. We don't allow this in MIR. Skip anything that's
     // already a successor.
     if (!AddedSuccessors.insert(Succ).second)
       continue;
     CurBB.addSuccessor(&getMBB(*Succ));
   }
 
   return true;
 }
 
 static bool isSwiftError(const Value *V) {
   if (auto Arg = dyn_cast<Argument>(V))
     return Arg->hasSwiftErrorAttr();
   if (auto AI = dyn_cast<AllocaInst>(V))
     return AI->isSwiftError();
   return false;
 }
 
 bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
   const LoadInst &LI = cast<LoadInst>(U);
   if (DL->getTypeStoreSize(LI.getType()) == 0)
     return true;
 
   ArrayRef<Register> Regs = getOrCreateVRegs(LI);
   ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
   Register Base = getOrCreateVReg(*LI.getPointerOperand());
 
   Type *OffsetIRTy = DL->getIntPtrType(LI.getPointerOperandType());
   LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
 
   if (CLI->supportSwiftError() && isSwiftError(LI.getPointerOperand())) {
     assert(Regs.size() == 1 && "swifterror should be single pointer");
     Register VReg = SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(),
                                                     LI.getPointerOperand());
     MIRBuilder.buildCopy(Regs[0], VReg);
     return true;
   }
 
   auto &TLI = *MF->getSubtarget().getTargetLowering();
   MachineMemOperand::Flags Flags = TLI.getLoadMemOperandFlags(LI, *DL);
 
   const MDNode *Ranges =
       Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
   for (unsigned i = 0; i < Regs.size(); ++i) {
     Register Addr;
     MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
 
     MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
     Align BaseAlign = getMemOpAlign(LI);
     auto MMO = MF->getMachineMemOperand(
         Ptr, Flags, MRI->getType(Regs[i]),
         commonAlignment(BaseAlign, Offsets[i] / 8), LI.getAAMetadata(), Ranges,
         LI.getSyncScopeID(), LI.getOrdering());
     MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
   }
 
   return true;
 }
 
 bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
   const StoreInst &SI = cast<StoreInst>(U);
   if (DL->getTypeStoreSize(SI.getValueOperand()->getType()) == 0)
     return true;
 
   ArrayRef<Register> Vals = getOrCreateVRegs(*SI.getValueOperand());
   ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
   Register Base = getOrCreateVReg(*SI.getPointerOperand());
 
   Type *OffsetIRTy = DL->getIntPtrType(SI.getPointerOperandType());
   LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
 
   if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) {
     assert(Vals.size() == 1 && "swifterror should be single pointer");
 
     Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
                                                     SI.getPointerOperand());
     MIRBuilder.buildCopy(VReg, Vals[0]);
     return true;
   }
 
   auto &TLI = *MF->getSubtarget().getTargetLowering();
   MachineMemOperand::Flags Flags = TLI.getStoreMemOperandFlags(SI, *DL);
 
   for (unsigned i = 0; i < Vals.size(); ++i) {
     Register Addr;
     MIRBuilder.materializePtrAdd(Addr, Base, OffsetTy, Offsets[i] / 8);
 
     MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
     Align BaseAlign = getMemOpAlign(SI);
     auto MMO = MF->getMachineMemOperand(
         Ptr, Flags, MRI->getType(Vals[i]),
         commonAlignment(BaseAlign, Offsets[i] / 8), SI.getAAMetadata(), nullptr,
         SI.getSyncScopeID(), SI.getOrdering());
     MIRBuilder.buildStore(Vals[i], Addr, *MMO);
   }
   return true;
 }
 
 static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
   const Value *Src = U.getOperand(0);
   Type *Int32Ty = Type::getInt32Ty(U.getContext());
 
   // getIndexedOffsetInType is designed for GEPs, so the first index is the
   // usual array element rather than looking into the actual aggregate.
   SmallVector<Value *, 1> Indices;
   Indices.push_back(ConstantInt::get(Int32Ty, 0));
 
   if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(&U)) {
     for (auto Idx : EVI->indices())
       Indices.push_back(ConstantInt::get(Int32Ty, Idx));
   } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {
     for (auto Idx : IVI->indices())
       Indices.push_back(ConstantInt::get(Int32Ty, Idx));
   } else {
     for (unsigned i = 1; i < U.getNumOperands(); ++i)
       Indices.push_back(U.getOperand(i));
   }
 
   return 8 * static_cast<uint64_t>(
                  DL.getIndexedOffsetInType(Src->getType(), Indices));
 }
 
 bool IRTranslator::translateExtractValue(const User &U,
                                          MachineIRBuilder &MIRBuilder) {
   const Value *Src = U.getOperand(0);
   uint64_t Offset = getOffsetFromIndices(U, *DL);
   ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
   ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*Src);
   unsigned Idx = llvm::lower_bound(Offsets, Offset) - Offsets.begin();
   auto &DstRegs = allocateVRegs(U);
 
   for (unsigned i = 0; i < DstRegs.size(); ++i)
     DstRegs[i] = SrcRegs[Idx++];
 
   return true;
 }
 
 bool IRTranslator::translateInsertValue(const User &U,
                                         MachineIRBuilder &MIRBuilder) {
   const Value *Src = U.getOperand(0);
   uint64_t Offset = getOffsetFromIndices(U, *DL);
   auto &DstRegs = allocateVRegs(U);
   ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);
   ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
   ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));
   auto InsertedIt = InsertedRegs.begin();
 
   for (unsigned i = 0; i < DstRegs.size(); ++i) {
     if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())
       DstRegs[i] = *InsertedIt++;
     else
       DstRegs[i] = SrcRegs[i];
   }
 
   return true;
 }
 
 bool IRTranslator::translateSelect(const User &U,
                                    MachineIRBuilder &MIRBuilder) {
   Register Tst = getOrCreateVReg(*U.getOperand(0));
   ArrayRef<Register> ResRegs = getOrCreateVRegs(U);
   ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
   ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
 
   uint16_t Flags = 0;
   if (const SelectInst *SI = dyn_cast<SelectInst>(&U))
     Flags = MachineInstr::copyFlagsFromInstruction(*SI);
 
   for (unsigned i = 0; i < ResRegs.size(); ++i) {
     MIRBuilder.buildSelect(ResRegs[i], Tst, Op0Regs[i], Op1Regs[i], Flags);
   }
 
   return true;
 }
 
 bool IRTranslator::translateCopy(const User &U, const Value &V,
                                  MachineIRBuilder &MIRBuilder) {
   Register Src = getOrCreateVReg(V);
   auto &Regs = *VMap.getVRegs(U);
   if (Regs.empty()) {
     Regs.push_back(Src);
     VMap.getOffsets(U)->push_back(0);
   } else {
     // If we already assigned a vreg for this instruction, we can't change that.
     // Emit a copy to satisfy the users we already emitted.
     MIRBuilder.buildCopy(Regs[0], Src);
   }
   return true;
 }
 
 bool IRTranslator::translateBitCast(const User &U,
                                     MachineIRBuilder &MIRBuilder) {
   // If we're bitcasting to the source type, we can reuse the source vreg.
   if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==
       getLLTForType(*U.getType(), *DL))
     return translateCopy(U, *U.getOperand(0), MIRBuilder);
 
   return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
 }
 
 bool IRTranslator::translateCast(unsigned Opcode, const User &U,
                                  MachineIRBuilder &MIRBuilder) {
   Register Op = getOrCreateVReg(*U.getOperand(0));
   Register Res = getOrCreateVReg(U);
   MIRBuilder.buildInstr(Opcode, {Res}, {Op});
   return true;
 }
 
 bool IRTranslator::translateGetElementPtr(const User &U,
                                           MachineIRBuilder &MIRBuilder) {
   Value &Op0 = *U.getOperand(0);
   Register BaseReg = getOrCreateVReg(Op0);
   Type *PtrIRTy = Op0.getType();
   LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
   Type *OffsetIRTy = DL->getIntPtrType(PtrIRTy);
   LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
 
   // Normalize Vector GEP - all scalar operands should be converted to the
   // splat vector.
   unsigned VectorWidth = 0;
 
   // True if we should use a splat vector; using VectorWidth alone is not
   // sufficient.
   bool WantSplatVector = false;
   if (auto *VT = dyn_cast<VectorType>(U.getType())) {
     VectorWidth = cast<FixedVectorType>(VT)->getNumElements();
     // We don't produce 1 x N vectors; those are treated as scalars.
     WantSplatVector = VectorWidth > 1;
   }
 
   // We might need to splat the base pointer into a vector if the offsets
   // are vectors.
   if (WantSplatVector && !PtrTy.isVector()) {
     BaseReg =
         MIRBuilder
             .buildSplatVector(LLT::fixed_vector(VectorWidth, PtrTy), BaseReg)
             .getReg(0);
     PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth);
     PtrTy = getLLTForType(*PtrIRTy, *DL);
     OffsetIRTy = DL->getIntPtrType(PtrIRTy);
     OffsetTy = getLLTForType(*OffsetIRTy, *DL);
   }
 
   int64_t Offset = 0;
   for (gep_type_iterator GTI = gep_type_begin(&U), E = gep_type_end(&U);
        GTI != E; ++GTI) {
     const Value *Idx = GTI.getOperand();
     if (StructType *StTy = GTI.getStructTypeOrNull()) {
       unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
       Offset += DL->getStructLayout(StTy)->getElementOffset(Field);
       continue;
     } else {
       uint64_t ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());
 
       // If this is a scalar constant or a splat vector of constants,
       // handle it quickly.
       if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
         Offset += ElementSize * CI->getSExtValue();
         continue;
       }
 
       if (Offset != 0) {
         auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset);
         BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, OffsetMIB.getReg(0))
                       .getReg(0);
         Offset = 0;
       }
 
       Register IdxReg = getOrCreateVReg(*Idx);
       LLT IdxTy = MRI->getType(IdxReg);
       if (IdxTy != OffsetTy) {
         if (!IdxTy.isVector() && WantSplatVector) {
           IdxReg = MIRBuilder.buildSplatVector(
             OffsetTy.changeElementType(IdxTy), IdxReg).getReg(0);
         }
 
         IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0);
       }
 
       // N = N + Idx * ElementSize;
       // Avoid doing it for ElementSize of 1.
       Register GepOffsetReg;
       if (ElementSize != 1) {
         auto ElementSizeMIB = MIRBuilder.buildConstant(
             getLLTForType(*OffsetIRTy, *DL), ElementSize);
         GepOffsetReg =
             MIRBuilder.buildMul(OffsetTy, IdxReg, ElementSizeMIB).getReg(0);
       } else
         GepOffsetReg = IdxReg;
 
       BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, GepOffsetReg).getReg(0);
     }
   }
 
   if (Offset != 0) {
     auto OffsetMIB =
         MIRBuilder.buildConstant(OffsetTy, Offset);
     MIRBuilder.buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0));
     return true;
   }
 
   MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
   return true;
 }
 
 bool IRTranslator::translateMemFunc(const CallInst &CI,
                                     MachineIRBuilder &MIRBuilder,
                                     unsigned Opcode) {
 
   // If the source is undef, then just emit a nop.
   if (isa<UndefValue>(CI.getArgOperand(1)))
     return true;
 
   SmallVector<Register, 3> SrcRegs;
 
   unsigned MinPtrSize = UINT_MAX;
   for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(AI) != AE; ++AI) {
     Register SrcReg = getOrCreateVReg(**AI);
     LLT SrcTy = MRI->getType(SrcReg);
     if (SrcTy.isPointer())
       MinPtrSize = std::min<unsigned>(SrcTy.getSizeInBits(), MinPtrSize);
     SrcRegs.push_back(SrcReg);
   }
 
   LLT SizeTy = LLT::scalar(MinPtrSize);
 
   // The size operand should be the minimum of the pointer sizes.
   Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];
   if (MRI->getType(SizeOpReg) != SizeTy)
     SizeOpReg = MIRBuilder.buildZExtOrTrunc(SizeTy, SizeOpReg).getReg(0);
 
   auto ICall = MIRBuilder.buildInstr(Opcode);
   for (Register SrcReg : SrcRegs)
     ICall.addUse(SrcReg);
 
   Align DstAlign;
   Align SrcAlign;
   unsigned IsVol =
       cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1))->getZExtValue();
 
   if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
     DstAlign = MCI->getDestAlign().valueOrOne();
     SrcAlign = MCI->getSourceAlign().valueOrOne();
   } else if (auto *MCI = dyn_cast<MemCpyInlineInst>(&CI)) {
     DstAlign = MCI->getDestAlign().valueOrOne();
     SrcAlign = MCI->getSourceAlign().valueOrOne();
   } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
     DstAlign = MMI->getDestAlign().valueOrOne();
     SrcAlign = MMI->getSourceAlign().valueOrOne();
   } else {
     auto *MSI = cast<MemSetInst>(&CI);
     DstAlign = MSI->getDestAlign().valueOrOne();
   }
 
   if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {
     // We need to propagate the tail call flag from the IR inst as an argument.
     // Otherwise, we have to pessimize and assume later that we cannot tail call
     // any memory intrinsics.
     ICall.addImm(CI.isTailCall() ? 1 : 0);
   }
 
   // Create mem operands to store the alignment and volatile info.
   auto VolFlag = IsVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
   ICall.addMemOperand(MF->getMachineMemOperand(
       MachinePointerInfo(CI.getArgOperand(0)),
       MachineMemOperand::MOStore | VolFlag, 1, DstAlign));
   if (Opcode != TargetOpcode::G_MEMSET)
     ICall.addMemOperand(MF->getMachineMemOperand(
         MachinePointerInfo(CI.getArgOperand(1)),
         MachineMemOperand::MOLoad | VolFlag, 1, SrcAlign));
 
   return true;
 }
 
 void IRTranslator::getStackGuard(Register DstReg,
                                  MachineIRBuilder &MIRBuilder) {
   const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
   MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF));
   auto MIB =
       MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD, {DstReg}, {});
 
   auto &TLI = *MF->getSubtarget().getTargetLowering();
   Value *Global = TLI.getSDagStackGuard(*MF->getFunction().getParent());
   if (!Global)
     return;
 
   unsigned AddrSpace = Global->getType()->getPointerAddressSpace();
   LLT PtrTy = LLT::pointer(AddrSpace, DL->getPointerSizeInBits(AddrSpace));
 
   MachinePointerInfo MPInfo(Global);
   auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
                MachineMemOperand::MODereferenceable;
   MachineMemOperand *MemRef = MF->getMachineMemOperand(
       MPInfo, Flags, PtrTy, DL->getPointerABIAlignment(AddrSpace));
   MIB.setMemRefs({MemRef});
 }
 
 bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
                                               MachineIRBuilder &MIRBuilder) {
   ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
   MIRBuilder.buildInstr(
       Op, {ResRegs[0], ResRegs[1]},
       {getOrCreateVReg(*CI.getOperand(0)), getOrCreateVReg(*CI.getOperand(1))});
 
   return true;
 }
 
 bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
                                                 MachineIRBuilder &MIRBuilder) {
   Register Dst = getOrCreateVReg(CI);
   Register Src0 = getOrCreateVReg(*CI.getOperand(0));
   Register Src1 = getOrCreateVReg(*CI.getOperand(1));
   uint64_t Scale = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
   MIRBuilder.buildInstr(Op, {Dst}, { Src0, Src1, Scale });
   return true;
 }
 
 unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
   switch (ID) {
     default:
       break;
     case Intrinsic::bswap:
       return TargetOpcode::G_BSWAP;
     case Intrinsic::bitreverse:
       return TargetOpcode::G_BITREVERSE;
     case Intrinsic::fshl:
       return TargetOpcode::G_FSHL;
     case Intrinsic::fshr:
       return TargetOpcode::G_FSHR;
     case Intrinsic::ceil:
       return TargetOpcode::G_FCEIL;
     case Intrinsic::cos:
       return TargetOpcode::G_FCOS;
     case Intrinsic::ctpop:
       return TargetOpcode::G_CTPOP;
     case Intrinsic::exp:
       return TargetOpcode::G_FEXP;
     case Intrinsic::exp2:
       return TargetOpcode::G_FEXP2;
     case Intrinsic::fabs:
       return TargetOpcode::G_FABS;
     case Intrinsic::copysign:
       return TargetOpcode::G_FCOPYSIGN;
     case Intrinsic::minnum:
       return TargetOpcode::G_FMINNUM;
     case Intrinsic::maxnum:
       return TargetOpcode::G_FMAXNUM;
     case Intrinsic::minimum:
       return TargetOpcode::G_FMINIMUM;
     case Intrinsic::maximum:
       return TargetOpcode::G_FMAXIMUM;
     case Intrinsic::canonicalize:
       return TargetOpcode::G_FCANONICALIZE;
     case Intrinsic::floor:
       return TargetOpcode::G_FFLOOR;
     case Intrinsic::fma:
       return TargetOpcode::G_FMA;
     case Intrinsic::log:
       return TargetOpcode::G_FLOG;
     case Intrinsic::log2:
       return TargetOpcode::G_FLOG2;
     case Intrinsic::log10:
       return TargetOpcode::G_FLOG10;
     case Intrinsic::nearbyint:
       return TargetOpcode::G_FNEARBYINT;
     case Intrinsic::pow:
       return TargetOpcode::G_FPOW;
     case Intrinsic::powi:
       return TargetOpcode::G_FPOWI;
     case Intrinsic::rint:
       return TargetOpcode::G_FRINT;
     case Intrinsic::round:
       return TargetOpcode::G_INTRINSIC_ROUND;
     case Intrinsic::roundeven:
       return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
     case Intrinsic::sin:
       return TargetOpcode::G_FSIN;
     case Intrinsic::sqrt:
       return TargetOpcode::G_FSQRT;
     case Intrinsic::trunc:
       return TargetOpcode::G_INTRINSIC_TRUNC;
     case Intrinsic::readcyclecounter:
       return TargetOpcode::G_READCYCLECOUNTER;
     case Intrinsic::ptrmask:
       return TargetOpcode::G_PTRMASK;
     case Intrinsic::lrint:
       return TargetOpcode::G_INTRINSIC_LRINT;
     // FADD/FMUL require checking the FMF, so are handled elsewhere.
     case Intrinsic::vector_reduce_fmin:
       return TargetOpcode::G_VECREDUCE_FMIN;
     case Intrinsic::vector_reduce_fmax:
       return TargetOpcode::G_VECREDUCE_FMAX;
     case Intrinsic::vector_reduce_add:
       return TargetOpcode::G_VECREDUCE_ADD;
     case Intrinsic::vector_reduce_mul:
       return TargetOpcode::G_VECREDUCE_MUL;
     case Intrinsic::vector_reduce_and:
       return TargetOpcode::G_VECREDUCE_AND;
     case Intrinsic::vector_reduce_or:
       return TargetOpcode::G_VECREDUCE_OR;
     case Intrinsic::vector_reduce_xor:
       return TargetOpcode::G_VECREDUCE_XOR;
     case Intrinsic::vector_reduce_smax:
       return TargetOpcode::G_VECREDUCE_SMAX;
     case Intrinsic::vector_reduce_smin:
       return TargetOpcode::G_VECREDUCE_SMIN;
     case Intrinsic::vector_reduce_umax:
       return TargetOpcode::G_VECREDUCE_UMAX;
     case Intrinsic::vector_reduce_umin:
       return TargetOpcode::G_VECREDUCE_UMIN;
     case Intrinsic::lround:
       return TargetOpcode::G_LROUND;
     case Intrinsic::llround:
       return TargetOpcode::G_LLROUND;
   }
   return Intrinsic::not_intrinsic;
 }
 
 bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
                                             Intrinsic::ID ID,
                                             MachineIRBuilder &MIRBuilder) {
 
   unsigned Op = getSimpleIntrinsicOpcode(ID);
 
   // Is this a simple intrinsic?
   if (Op == Intrinsic::not_intrinsic)
     return false;
 
   // Yes. Let's translate it.
   SmallVector<llvm::SrcOp, 4> VRegs;
   for (auto &Arg : CI.args())
     VRegs.push_back(getOrCreateVReg(*Arg));
 
   MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
                         MachineInstr::copyFlagsFromInstruction(CI));
   return true;
 }
 
 // TODO: Include ConstainedOps.def when all strict instructions are defined.
 static unsigned getConstrainedOpcode(Intrinsic::ID ID) {
   switch (ID) {
   case Intrinsic::experimental_constrained_fadd:
     return TargetOpcode::G_STRICT_FADD;
   case Intrinsic::experimental_constrained_fsub:
     return TargetOpcode::G_STRICT_FSUB;
   case Intrinsic::experimental_constrained_fmul:
     return TargetOpcode::G_STRICT_FMUL;
   case Intrinsic::experimental_constrained_fdiv:
     return TargetOpcode::G_STRICT_FDIV;
   case Intrinsic::experimental_constrained_frem:
     return TargetOpcode::G_STRICT_FREM;
   case Intrinsic::experimental_constrained_fma:
     return TargetOpcode::G_STRICT_FMA;
   case Intrinsic::experimental_constrained_sqrt:
     return TargetOpcode::G_STRICT_FSQRT;
   default:
     return 0;
   }
 }
 
 bool IRTranslator::translateConstrainedFPIntrinsic(
   const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
   fp::ExceptionBehavior EB = FPI.getExceptionBehavior().getValue();
 
   unsigned Opcode = getConstrainedOpcode(FPI.getIntrinsicID());
   if (!Opcode)
     return false;
 
   unsigned Flags = MachineInstr::copyFlagsFromInstruction(FPI);
   if (EB == fp::ExceptionBehavior::ebIgnore)
     Flags |= MachineInstr::NoFPExcept;
 
   SmallVector<llvm::SrcOp, 4> VRegs;
   VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(0)));
   if (!FPI.isUnaryOp())
     VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(1)));
   if (FPI.isTernaryOp())
     VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(2)));
 
   MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(FPI)}, VRegs, Flags);
   return true;
 }
 
 bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
                                            MachineIRBuilder &MIRBuilder) {
   if (auto *MI = dyn_cast<AnyMemIntrinsic>(&CI)) {
     if (ORE->enabled()) {
       const Function &F = *MI->getParent()->getParent();
       auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
       if (MemoryOpRemark::canHandle(MI, TLI)) {
         MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, TLI);
         R.visit(MI);
       }
     }
   }
 
   // If this is a simple intrinsic (that is, we just need to add a def of
   // a vreg, and uses for each arg operand, then translate it.
   if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
     return true;
 
   switch (ID) {
   default:
     break;
   case Intrinsic::lifetime_start:
   case Intrinsic::lifetime_end: {
     // No stack colouring in O0, discard region information.
     if (MF->getTarget().getOptLevel() == CodeGenOpt::None)
       return true;
 
     unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
                                                   : TargetOpcode::LIFETIME_END;
 
     // Get the underlying objects for the location passed on the lifetime
     // marker.
     SmallVector<const Value *, 4> Allocas;
     getUnderlyingObjects(CI.getArgOperand(1), Allocas);
 
     // Iterate over each underlying object, creating lifetime markers for each
     // static alloca. Quit if we find a non-static alloca.
     for (const Value *V : Allocas) {
       const AllocaInst *AI = dyn_cast<AllocaInst>(V);
       if (!AI)
         continue;
 
       if (!AI->isStaticAlloca())
         return true;
 
       MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI));
     }
     return true;
   }
   case Intrinsic::dbg_declare: {
     const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);
     assert(DI.getVariable() && "Missing variable");
 
     const Value *Address = DI.getAddress();
     if (!Address || isa<UndefValue>(Address)) {
       LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
       return true;
     }
 
     assert(DI.getVariable()->isValidLocationForIntrinsic(
                MIRBuilder.getDebugLoc()) &&
            "Expected inlined-at fields to agree");
     auto AI = dyn_cast<AllocaInst>(Address);
     if (AI && AI->isStaticAlloca()) {
       // Static allocas are tracked at the MF level, no need for DBG_VALUE
       // instructions (in fact, they get ignored if they *do* exist).
       MF->setVariableDbgInfo(DI.getVariable(), DI.getExpression(),
                              getOrCreateFrameIndex(*AI), DI.getDebugLoc());
     } else {
       // A dbg.declare describes the address of a source variable, so lower it
       // into an indirect DBG_VALUE.
       MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
                                        DI.getVariable(), DI.getExpression());
     }
     return true;
   }
   case Intrinsic::dbg_label: {
     const DbgLabelInst &DI = cast<DbgLabelInst>(CI);
     assert(DI.getLabel() && "Missing label");
 
     assert(DI.getLabel()->isValidLocationForIntrinsic(
                MIRBuilder.getDebugLoc()) &&
            "Expected inlined-at fields to agree");
 
     MIRBuilder.buildDbgLabel(DI.getLabel());
     return true;
   }
   case Intrinsic::vaend:
     // No target I know of cares about va_end. Certainly no in-tree target
     // does. Simplest intrinsic ever!
     return true;
   case Intrinsic::vastart: {
     auto &TLI = *MF->getSubtarget().getTargetLowering();
     Value *Ptr = CI.getArgOperand(0);
     unsigned ListSize = TLI.getVaListSizeInBits(*DL) / 8;
 
     // FIXME: Get alignment
     MIRBuilder.buildInstr(TargetOpcode::G_VASTART, {}, {getOrCreateVReg(*Ptr)})
         .addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Ptr),
                                                 MachineMemOperand::MOStore,
                                                 ListSize, Align(1)));
     return true;
   }
   case Intrinsic::dbg_value: {
     // This form of DBG_VALUE is target-independent.
     const DbgValueInst &DI = cast<DbgValueInst>(CI);
     const Value *V = DI.getValue();
     assert(DI.getVariable()->isValidLocationForIntrinsic(
                MIRBuilder.getDebugLoc()) &&
            "Expected inlined-at fields to agree");
     if (!V || DI.hasArgList()) {
       // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to
       // terminate any prior location.
       MIRBuilder.buildIndirectDbgValue(0, DI.getVariable(), DI.getExpression());
     } else if (const auto *CI = dyn_cast<Constant>(V)) {
       MIRBuilder.buildConstDbgValue(*CI, DI.getVariable(), DI.getExpression());
     } else {
       for (Register Reg : getOrCreateVRegs(*V)) {
         // FIXME: This does not handle register-indirect values at offset 0. The
         // direct/indirect thing shouldn't really be handled by something as
         // implicit as reg+noreg vs reg+imm in the first place, but it seems
         // pretty baked in right now.
         MIRBuilder.buildDirectDbgValue(Reg, DI.getVariable(), DI.getExpression());
       }
     }
     return true;
   }
   case Intrinsic::uadd_with_overflow:
     return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);
   case Intrinsic::sadd_with_overflow:
     return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
   case Intrinsic::usub_with_overflow:
     return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);
   case Intrinsic::ssub_with_overflow:
     return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
   case Intrinsic::umul_with_overflow:
     return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
   case Intrinsic::smul_with_overflow:
     return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
   case Intrinsic::uadd_sat:
     return translateBinaryOp(TargetOpcode::G_UADDSAT, CI, MIRBuilder);
   case Intrinsic::sadd_sat:
     return translateBinaryOp(TargetOpcode::G_SADDSAT, CI, MIRBuilder);
   case Intrinsic::usub_sat:
     return translateBinaryOp(TargetOpcode::G_USUBSAT, CI, MIRBuilder);
   case Intrinsic::ssub_sat:
     return translateBinaryOp(TargetOpcode::G_SSUBSAT, CI, MIRBuilder);
   case Intrinsic::ushl_sat:
     return translateBinaryOp(TargetOpcode::G_USHLSAT, CI, MIRBuilder);
   case Intrinsic::sshl_sat:
     return translateBinaryOp(TargetOpcode::G_SSHLSAT, CI, MIRBuilder);
   case Intrinsic::umin:
     return translateBinaryOp(TargetOpcode::G_UMIN, CI, MIRBuilder);
   case Intrinsic::umax:
     return translateBinaryOp(TargetOpcode::G_UMAX, CI, MIRBuilder);
   case Intrinsic::smin:
     return translateBinaryOp(TargetOpcode::G_SMIN, CI, MIRBuilder);
   case Intrinsic::smax:
     return translateBinaryOp(TargetOpcode::G_SMAX, CI, MIRBuilder);
   case Intrinsic::abs:
     // TODO: Preserve "int min is poison" arg in GMIR?
     return translateUnaryOp(TargetOpcode::G_ABS, CI, MIRBuilder);
   case Intrinsic::smul_fix:
     return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIX, CI, MIRBuilder);
   case Intrinsic::umul_fix:
     return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIX, CI, MIRBuilder);
   case Intrinsic::smul_fix_sat:
     return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
   case Intrinsic::umul_fix_sat:
     return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
   case Intrinsic::sdiv_fix:
     return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
   case Intrinsic::udiv_fix:
     return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
   case Intrinsic::sdiv_fix_sat:
     return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
   case Intrinsic::udiv_fix_sat:
     return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
   case Intrinsic::fmuladd: {
     const TargetMachine &TM = MF->getTarget();
     const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
     Register Dst = getOrCreateVReg(CI);
     Register Op0 = getOrCreateVReg(*CI.getArgOperand(0));
     Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));
     Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));
     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
         TLI.isFMAFasterThanFMulAndFAdd(*MF,
                                        TLI.getValueType(*DL, CI.getType()))) {
       // TODO: Revisit this to see if we should move this part of the
       // lowering to the combiner.
       MIRBuilder.buildFMA(Dst, Op0, Op1, Op2,
                           MachineInstr::copyFlagsFromInstruction(CI));
     } else {
       LLT Ty = getLLTForType(*CI.getType(), *DL);
       auto FMul = MIRBuilder.buildFMul(
           Ty, Op0, Op1, MachineInstr::copyFlagsFromInstruction(CI));
       MIRBuilder.buildFAdd(Dst, FMul, Op2,
                            MachineInstr::copyFlagsFromInstruction(CI));
     }
     return true;
   }
   case Intrinsic::convert_from_fp16:
     // FIXME: This intrinsic should probably be removed from the IR.
     MIRBuilder.buildFPExt(getOrCreateVReg(CI),
                           getOrCreateVReg(*CI.getArgOperand(0)),
                           MachineInstr::copyFlagsFromInstruction(CI));
     return true;
   case Intrinsic::convert_to_fp16:
     // FIXME: This intrinsic should probably be removed from the IR.
     MIRBuilder.buildFPTrunc(getOrCreateVReg(CI),
                             getOrCreateVReg(*CI.getArgOperand(0)),
                             MachineInstr::copyFlagsFromInstruction(CI));
     return true;
   case Intrinsic::memcpy_inline:
     return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY_INLINE);
   case Intrinsic::memcpy:
     return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY);
   case Intrinsic::memmove:
     return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMMOVE);
   case Intrinsic::memset:
     return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET);
   case Intrinsic::eh_typeid_for: {
     GlobalValue *GV = ExtractTypeInfo(CI.getArgOperand(0));
     Register Reg = getOrCreateVReg(CI);
     unsigned TypeID = MF->getTypeIDFor(GV);
     MIRBuilder.buildConstant(Reg, TypeID);
     return true;
   }
   case Intrinsic::objectsize:
     llvm_unreachable("llvm.objectsize.* should have been lowered already");
 
   case Intrinsic::is_constant:
     llvm_unreachable("llvm.is.constant.* should have been lowered already");
 
   case Intrinsic::stackguard:
     getStackGuard(getOrCreateVReg(CI), MIRBuilder);
     return true;
   case Intrinsic::stackprotector: {
     LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
     Register GuardVal = MRI->createGenericVirtualRegister(PtrTy);
     getStackGuard(GuardVal, MIRBuilder);
 
     AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));
     int FI = getOrCreateFrameIndex(*Slot);
     MF->getFrameInfo().setStackProtectorIndex(FI);
 
     MIRBuilder.buildStore(
         GuardVal, getOrCreateVReg(*Slot),
         *MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI),
                                   MachineMemOperand::MOStore |
                                       MachineMemOperand::MOVolatile,
                                   PtrTy, Align(8)));
     return true;
   }
   case Intrinsic::stacksave: {
     // Save the stack pointer to the location provided by the intrinsic.
     Register Reg = getOrCreateVReg(CI);
     Register StackPtr = MF->getSubtarget()
                             .getTargetLowering()
                             ->getStackPointerRegisterToSaveRestore();
 
     // If the target doesn't specify a stack pointer, then fall back.
     if (!StackPtr)
       return false;
 
     MIRBuilder.buildCopy(Reg, StackPtr);
     return true;
   }
   case Intrinsic::stackrestore: {
     // Restore the stack pointer from the location provided by the intrinsic.
     Register Reg = getOrCreateVReg(*CI.getArgOperand(0));
     Register StackPtr = MF->getSubtarget()
                             .getTargetLowering()
                             ->getStackPointerRegisterToSaveRestore();
 
     // If the target doesn't specify a stack pointer, then fall back.
     if (!StackPtr)
       return false;
 
     MIRBuilder.buildCopy(StackPtr, Reg);
     return true;
   }
   case Intrinsic::cttz:
   case Intrinsic::ctlz: {
     ConstantInt *Cst = cast<ConstantInt>(CI.getArgOperand(1));
     bool isTrailing = ID == Intrinsic::cttz;
     unsigned Opcode = isTrailing
                           ? Cst->isZero() ? TargetOpcode::G_CTTZ
                                           : TargetOpcode::G_CTTZ_ZERO_UNDEF
                           : Cst->isZero() ? TargetOpcode::G_CTLZ
                                           : TargetOpcode::G_CTLZ_ZERO_UNDEF;
     MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(CI)},
                           {getOrCreateVReg(*CI.getArgOperand(0))});
     return true;
   }
   case Intrinsic::invariant_start: {
     LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
     Register Undef = MRI->createGenericVirtualRegister(PtrTy);
     MIRBuilder.buildUndef(Undef);
     return true;
   }
   case Intrinsic::invariant_end:
     return true;
   case Intrinsic::expect:
   case Intrinsic::annotation:
   case Intrinsic::ptr_annotation:
   case Intrinsic::launder_invariant_group:
   case Intrinsic::strip_invariant_group: {
     // Drop the intrinsic, but forward the value.
     MIRBuilder.buildCopy(getOrCreateVReg(CI),
                          getOrCreateVReg(*CI.getArgOperand(0)));
     return true;
   }
   case Intrinsic::assume:
   case Intrinsic::experimental_noalias_scope_decl:
   case Intrinsic::var_annotation:
   case Intrinsic::sideeffect:
     // Discard annotate attributes, assumptions, and artificial side-effects.
     return true;
   case Intrinsic::read_volatile_register:
   case Intrinsic::read_register: {
     Value *Arg = CI.getArgOperand(0);
     MIRBuilder
         .buildInstr(TargetOpcode::G_READ_REGISTER, {getOrCreateVReg(CI)}, {})
         .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()));
     return true;
   }
   case Intrinsic::write_register: {
     Value *Arg = CI.getArgOperand(0);
     MIRBuilder.buildInstr(TargetOpcode::G_WRITE_REGISTER)
       .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()))
       .addUse(getOrCreateVReg(*CI.getArgOperand(1)));
     return true;
   }
   case Intrinsic::localescape: {
     MachineBasicBlock &EntryMBB = MF->front();
     StringRef EscapedName = GlobalValue::dropLLVMManglingEscape(MF->getName());
 
     // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
     // is the same on all targets.
     for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {
       Value *Arg = CI.getArgOperand(Idx)->stripPointerCasts();
       if (isa<ConstantPointerNull>(Arg))
         continue; // Skip null pointers. They represent a hole in index space.
 
       int FI = getOrCreateFrameIndex(*cast<AllocaInst>(Arg));
       MCSymbol *FrameAllocSym =
           MF->getMMI().getContext().getOrCreateFrameAllocSymbol(EscapedName,
                                                                 Idx);
 
       // This should be inserted at the start of the entry block.
       auto LocalEscape =
           MIRBuilder.buildInstrNoInsert(TargetOpcode::LOCAL_ESCAPE)
               .addSym(FrameAllocSym)
               .addFrameIndex(FI);
 
       EntryMBB.insert(EntryMBB.begin(), LocalEscape);
     }
 
     return true;
   }
   case Intrinsic::vector_reduce_fadd:
   case Intrinsic::vector_reduce_fmul: {
     // Need to check for the reassoc flag to decide whether we want a
     // sequential reduction opcode or not.
     Register Dst = getOrCreateVReg(CI);
     Register ScalarSrc = getOrCreateVReg(*CI.getArgOperand(0));
     Register VecSrc = getOrCreateVReg(*CI.getArgOperand(1));
     unsigned Opc = 0;
     if (!CI.hasAllowReassoc()) {
       // The sequential ordering case.
       Opc = ID == Intrinsic::vector_reduce_fadd
                 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
                 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
       MIRBuilder.buildInstr(Opc, {Dst}, {ScalarSrc, VecSrc},
                             MachineInstr::copyFlagsFromInstruction(CI));
       return true;
     }
     // We split the operation into a separate G_FADD/G_FMUL + the reduce,
     // since the associativity doesn't matter.
     unsigned ScalarOpc;
     if (ID == Intrinsic::vector_reduce_fadd) {
       Opc = TargetOpcode::G_VECREDUCE_FADD;
       ScalarOpc = TargetOpcode::G_FADD;
     } else {
       Opc = TargetOpcode::G_VECREDUCE_FMUL;
       ScalarOpc = TargetOpcode::G_FMUL;
     }
     LLT DstTy = MRI->getType(Dst);
     auto Rdx = MIRBuilder.buildInstr(
         Opc, {DstTy}, {VecSrc}, MachineInstr::copyFlagsFromInstruction(CI));
     MIRBuilder.buildInstr(ScalarOpc, {Dst}, {ScalarSrc, Rdx},
                           MachineInstr::copyFlagsFromInstruction(CI));
 
     return true;
   }
   case Intrinsic::trap:
   case Intrinsic::debugtrap:
   case Intrinsic::ubsantrap: {
     StringRef TrapFuncName =
         CI.getAttributes().getFnAttr("trap-func-name").getValueAsString();
     if (TrapFuncName.empty())
       break; // Use the default handling.
     CallLowering::CallLoweringInfo Info;
     if (ID == Intrinsic::ubsantrap) {
       Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)),
                                CI.getArgOperand(0)->getType(), 0});
     }
     Info.Callee = MachineOperand::CreateES(TrapFuncName.data());
     Info.CB = &CI;
     Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0};
     return CLI->lowerCall(MIRBuilder, Info);
   }
 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)  \
   case Intrinsic::INTRINSIC:
 #include "llvm/IR/ConstrainedOps.def"
     return translateConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(CI),
                                            MIRBuilder);
 
   }
   return false;
 }
 
 bool IRTranslator::translateInlineAsm(const CallBase &CB,
                                       MachineIRBuilder &MIRBuilder) {
 
   const InlineAsmLowering *ALI = MF->getSubtarget().getInlineAsmLowering();
 
   if (!ALI) {
     LLVM_DEBUG(
         dbgs() << "Inline asm lowering is not supported for this target yet\n");
     return false;
   }
 
   return ALI->lowerInlineAsm(
       MIRBuilder, CB, [&](const Value &Val) { return getOrCreateVRegs(Val); });
 }
 
 bool IRTranslator::translateCallBase(const CallBase &CB,
                                      MachineIRBuilder &MIRBuilder) {
   ArrayRef<Register> Res = getOrCreateVRegs(CB);
 
   SmallVector<ArrayRef<Register>, 8> Args;
   Register SwiftInVReg = 0;
   Register SwiftErrorVReg = 0;
   for (auto &Arg : CB.args()) {
     if (CLI->supportSwiftError() && isSwiftError(Arg)) {
       assert(SwiftInVReg == 0 && "Expected only one swift error argument");
       LLT Ty = getLLTForType(*Arg->getType(), *DL);
       SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
       MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
                                             &CB, &MIRBuilder.getMBB(), Arg));
       Args.emplace_back(makeArrayRef(SwiftInVReg));
       SwiftErrorVReg =
           SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);
       continue;
     }
     Args.push_back(getOrCreateVRegs(*Arg));
   }
 
   if (auto *CI = dyn_cast<CallInst>(&CB)) {
     if (ORE->enabled()) {
       const Function &F = *CI->getParent()->getParent();
       auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
       if (MemoryOpRemark::canHandle(CI, TLI)) {
         MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, TLI);
         R.visit(CI);
       }
     }
   }
 
   // We don't set HasCalls on MFI here yet because call lowering may decide to
   // optimize into tail calls. Instead, we defer that to selection where a final
   // scan is done to check if any instructions are calls.
   bool Success =
       CLI->lowerCall(MIRBuilder, CB, Res, Args, SwiftErrorVReg,
                      [&]() { return getOrCreateVReg(*CB.getCalledOperand()); });
 
   // Check if we just inserted a tail call.
   if (Success) {
     assert(!HasTailCall && "Can't tail call return twice from block?");
     const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
     HasTailCall = TII->isTailCall(*std::prev(MIRBuilder.getInsertPt()));
   }
 
   return Success;
 }
 
 bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
   const CallInst &CI = cast<CallInst>(U);
   auto TII = MF->getTarget().getIntrinsicInfo();
   const Function *F = CI.getCalledFunction();
 
   // FIXME: support Windows dllimport function calls.
   if (F && (F->hasDLLImportStorageClass() ||
             (MF->getTarget().getTargetTriple().isOSWindows() &&
              F->hasExternalWeakLinkage())))
     return false;
 
   // FIXME: support control flow guard targets.
   if (CI.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
     return false;
 
   if (CI.isInlineAsm())
     return translateInlineAsm(CI, MIRBuilder);
 
   diagnoseDontCall(CI);
 
   Intrinsic::ID ID = Intrinsic::not_intrinsic;
   if (F && F->isIntrinsic()) {
     ID = F->getIntrinsicID();
     if (TII && ID == Intrinsic::not_intrinsic)
       ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
   }
 
   if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
     return translateCallBase(CI, MIRBuilder);
 
   assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
 
   if (translateKnownIntrinsic(CI, ID, MIRBuilder))
     return true;
 
   ArrayRef<Register> ResultRegs;
   if (!CI.getType()->isVoidTy())
     ResultRegs = getOrCreateVRegs(CI);
 
   // Ignore the callsite attributes. Backend code is most likely not expecting
   // an intrinsic to sometimes have side effects and sometimes not.
   MachineInstrBuilder MIB =
       MIRBuilder.buildIntrinsic(ID, ResultRegs, !F->doesNotAccessMemory());
   if (isa<FPMathOperator>(CI))
     MIB->copyIRFlags(CI);
 
   for (auto &Arg : enumerate(CI.args())) {
     // If this is required to be an immediate, don't materialize it in a
     // register.
     if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
       if (ConstantInt *CI = dyn_cast<ConstantInt>(Arg.value())) {
         // imm arguments are more convenient than cimm (and realistically
         // probably sufficient), so use them.
         assert(CI->getBitWidth() <= 64 &&
                "large intrinsic immediates not handled");
         MIB.addImm(CI->getSExtValue());
       } else {
         MIB.addFPImm(cast<ConstantFP>(Arg.value()));
       }
     } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Arg.value())) {
       auto *MD = MDVal->getMetadata();
       auto *MDN = dyn_cast<MDNode>(MD);
       if (!MDN) {
         if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(MD))
           MDN = MDNode::get(MF->getFunction().getContext(), ConstMD);
         else // This was probably an MDString.
           return false;
       }
       MIB.addMetadata(MDN);
     } else {
       ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg.value());
       if (VRegs.size() > 1)
         return false;
       MIB.addUse(VRegs[0]);
     }
   }
 
   // Add a MachineMemOperand if it is a target mem intrinsic.
   const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
   TargetLowering::IntrinsicInfo Info;
   // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
   if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
     Align Alignment = Info.align.getValueOr(
         DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext())));
     LLT MemTy = Info.memVT.isSimple()
                     ? getLLTForMVT(Info.memVT.getSimpleVT())
                     : LLT::scalar(Info.memVT.getStoreSizeInBits());
     MIB.addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Info.ptrVal),
                                                Info.flags, MemTy, Alignment));
   }
 
   return true;
 }
 
 bool IRTranslator::findUnwindDestinations(
     const BasicBlock *EHPadBB,
     BranchProbability Prob,
     SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
         &UnwindDests) {
   EHPersonality Personality = classifyEHPersonality(
       EHPadBB->getParent()->getFunction().getPersonalityFn());
   bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
   bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
   bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
   bool IsSEH = isAsynchronousEHPersonality(Personality);
 
   if (IsWasmCXX) {
     // Ignore this for now.
     return false;
   }
 
   while (EHPadBB) {
     const Instruction *Pad = EHPadBB->getFirstNonPHI();
     BasicBlock *NewEHPadBB = nullptr;
     if (isa<LandingPadInst>(Pad)) {
       // Stop on landingpads. They are not funclets.
       UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
       break;
     }
     if (isa<CleanupPadInst>(Pad)) {
       // Stop on cleanup pads. Cleanups are always funclet entries for all known
       // personalities.
       UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);
       UnwindDests.back().first->setIsEHScopeEntry();
       UnwindDests.back().first->setIsEHFuncletEntry();
       break;
     }
     if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
       // Add the catchpad handlers to the possible destinations.
       for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
         UnwindDests.emplace_back(&getMBB(*CatchPadBB), Prob);
         // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
         if (IsMSVCCXX || IsCoreCLR)
           UnwindDests.back().first->setIsEHFuncletEntry();
         if (!IsSEH)
           UnwindDests.back().first->setIsEHScopeEntry();
       }
       NewEHPadBB = CatchSwitch->getUnwindDest();
     } else {
       continue;
     }
 
     BranchProbabilityInfo *BPI = FuncInfo.BPI;
     if (BPI && NewEHPadBB)
       Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
     EHPadBB = NewEHPadBB;
   }
   return true;
 }
 
 bool IRTranslator::translateInvoke(const User &U,
                                    MachineIRBuilder &MIRBuilder) {
   const InvokeInst &I = cast<InvokeInst>(U);
   MCContext &Context = MF->getContext();
 
   const BasicBlock *ReturnBB = I.getSuccessor(0);
   const BasicBlock *EHPadBB = I.getSuccessor(1);
 
   const Function *Fn = I.getCalledFunction();
 
   // FIXME: support invoking patchpoint and statepoint intrinsics.
   if (Fn && Fn->isIntrinsic())
     return false;
 
   // FIXME: support whatever these are.
   if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
     return false;
 
   // FIXME: support control flow guard targets.
   if (I.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
     return false;
 
   // FIXME: support Windows exception handling.
   if (!isa<LandingPadInst>(EHPadBB->getFirstNonPHI()))
     return false;
 
   bool LowerInlineAsm = I.isInlineAsm();
   bool NeedEHLabel = true;
   // If it can't throw then use a fast-path without emitting EH labels.
   if (LowerInlineAsm)
     NeedEHLabel = (cast<InlineAsm>(I.getCalledOperand()))->canThrow();
 
   // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
   // the region covered by the try.
   MCSymbol *BeginSymbol = nullptr;
   if (NeedEHLabel) {
     BeginSymbol = Context.createTempSymbol();
     MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
   }
 
   if (LowerInlineAsm) {
     if (!translateInlineAsm(I, MIRBuilder))
       return false;
   } else if (!translateCallBase(I, MIRBuilder))
     return false;
 
   MCSymbol *EndSymbol = nullptr;
   if (NeedEHLabel) {
     EndSymbol = Context.createTempSymbol();
     MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);
   }
 
   SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
   BranchProbabilityInfo *BPI = FuncInfo.BPI;
   MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();
   BranchProbability EHPadBBProb =
       BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
           : BranchProbability::getZero();
 
   if (!findUnwindDestinations(EHPadBB, EHPadBBProb, UnwindDests))
     return false;
 
   MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),
                     &ReturnMBB = getMBB(*ReturnBB);
   // Update successor info.
   addSuccessorWithProb(InvokeMBB, &ReturnMBB);
   for (auto &UnwindDest : UnwindDests) {
     UnwindDest.first->setIsEHPad();
     addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
   }
   InvokeMBB->normalizeSuccProbs();
 
   if (NeedEHLabel) {
     assert(BeginSymbol && "Expected a begin symbol!");
     assert(EndSymbol && "Expected an end symbol!");
     MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
   }
 
   MIRBuilder.buildBr(ReturnMBB);
   return true;
 }
 
 bool IRTranslator::translateCallBr(const User &U,
                                    MachineIRBuilder &MIRBuilder) {
   // FIXME: Implement this.
   return false;
 }
 
 bool IRTranslator::translateLandingPad(const User &U,
                                        MachineIRBuilder &MIRBuilder) {
   const LandingPadInst &LP = cast<LandingPadInst>(U);
 
   MachineBasicBlock &MBB = MIRBuilder.getMBB();
 
   MBB.setIsEHPad();
 
   // If there aren't registers to copy the values into (e.g., during SjLj
   // exceptions), then don't bother.
   auto &TLI = *MF->getSubtarget().getTargetLowering();
   const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
   if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
       TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
     return true;
 
   // If landingpad's return type is token type, we don't create DAG nodes
   // for its exception pointer and selector value. The extraction of exception
   // pointer or selector value from token type landingpads is not currently
   // supported.
   if (LP.getType()->isTokenTy())
     return true;
 
   // Add a label to mark the beginning of the landing pad.  Deletion of the
   // landing pad can thus be detected via the MachineModuleInfo.
   MIRBuilder.buildInstr(TargetOpcode::EH_LABEL)
     .addSym(MF->addLandingPad(&MBB));
 
   // If the unwinder does not preserve all registers, ensure that the
   // function marks the clobbered registers as used.
   const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
   if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
     MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask);
 
   LLT Ty = getLLTForType(*LP.getType(), *DL);
   Register Undef = MRI->createGenericVirtualRegister(Ty);
   MIRBuilder.buildUndef(Undef);
 
   SmallVector<LLT, 2> Tys;
   for (Type *Ty : cast<StructType>(LP.getType())->elements())
     Tys.push_back(getLLTForType(*Ty, *DL));
   assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
 
   // Mark exception register as live in.
   Register ExceptionReg = TLI.getExceptionPointerRegister(PersonalityFn);
   if (!ExceptionReg)
     return false;
 
   MBB.addLiveIn(ExceptionReg);
   ArrayRef<Register> ResRegs = getOrCreateVRegs(LP);
   MIRBuilder.buildCopy(ResRegs[0], ExceptionReg);
 
   Register SelectorReg = TLI.getExceptionSelectorRegister(PersonalityFn);
   if (!SelectorReg)
     return false;
 
   MBB.addLiveIn(SelectorReg);
   Register PtrVReg = MRI->createGenericVirtualRegister(Tys[0]);
   MIRBuilder.buildCopy(PtrVReg, SelectorReg);
   MIRBuilder.buildCast(ResRegs[1], PtrVReg);
 
   return true;
 }
 
 bool IRTranslator::translateAlloca(const User &U,
                                    MachineIRBuilder &MIRBuilder) {
   auto &AI = cast<AllocaInst>(U);
 
   if (AI.isSwiftError())
     return true;
 
   if (AI.isStaticAlloca()) {
     Register Res = getOrCreateVReg(AI);
     int FI = getOrCreateFrameIndex(AI);
     MIRBuilder.buildFrameIndex(Res, FI);
     return true;
   }
 
   // FIXME: support stack probing for Windows.
   if (MF->getTarget().getTargetTriple().isOSWindows())
     return false;
 
   // Now we're in the harder dynamic case.
   Register NumElts = getOrCreateVReg(*AI.getArraySize());
   Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
   LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);
   if (MRI->getType(NumElts) != IntPtrTy) {
     Register ExtElts = MRI->createGenericVirtualRegister(IntPtrTy);
     MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts);
     NumElts = ExtElts;
   }
 
   Type *Ty = AI.getAllocatedType();
 
   Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);
   Register TySize =
       getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, DL->getTypeAllocSize(Ty)));
   MIRBuilder.buildMul(AllocSize, NumElts, TySize);
 
   // Round the size of the allocation up to the stack alignment size
   // by add SA-1 to the size. This doesn't overflow because we're computing
   // an address inside an alloca.
   Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();
   auto SAMinusOne = MIRBuilder.buildConstant(IntPtrTy, StackAlign.value() - 1);
   auto AllocAdd = MIRBuilder.buildAdd(IntPtrTy, AllocSize, SAMinusOne,
                                       MachineInstr::NoUWrap);
   auto AlignCst =
       MIRBuilder.buildConstant(IntPtrTy, ~(uint64_t)(StackAlign.value() - 1));
   auto AlignedAlloc = MIRBuilder.buildAnd(IntPtrTy, AllocAdd, AlignCst);
 
   Align Alignment = std::max(AI.getAlign(), DL->getPrefTypeAlign(Ty));
   if (Alignment <= StackAlign)
     Alignment = Align(1);
   MIRBuilder.buildDynStackAlloc(getOrCreateVReg(AI), AlignedAlloc, Alignment);
 
   MF->getFrameInfo().CreateVariableSizedObject(Alignment, &AI);
   assert(MF->getFrameInfo().hasVarSizedObjects());
   return true;
 }
 
 bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
   // FIXME: We may need more info about the type. Because of how LLT works,
   // we're completely discarding the i64/double distinction here (amongst
   // others). Fortunately the ABIs I know of where that matters don't use va_arg
   // anyway but that's not guaranteed.
   MIRBuilder.buildInstr(TargetOpcode::G_VAARG, {getOrCreateVReg(U)},
                         {getOrCreateVReg(*U.getOperand(0)),
                          DL->getABITypeAlign(U.getType()).value()});
   return true;
 }
 
 bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuilder) {
     if (!MF->getTarget().Options.TrapUnreachable)
     return true;
 
   auto &UI = cast<UnreachableInst>(U);
   // We may be able to ignore unreachable behind a noreturn call.
   if (MF->getTarget().Options.NoTrapAfterNoreturn) {
     const BasicBlock &BB = *UI.getParent();
     if (&UI != &BB.front()) {
       BasicBlock::const_iterator PredI =
         std::prev(BasicBlock::const_iterator(UI));
       if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
         if (Call->doesNotReturn())
           return true;
       }
     }
   }
 
   MIRBuilder.buildIntrinsic(Intrinsic::trap, ArrayRef<Register>(), true);
   return true;
 }
 
 bool IRTranslator::translateInsertElement(const User &U,
                                           MachineIRBuilder &MIRBuilder) {
   // If it is a <1 x Ty> vector, use the scalar as it is
   // not a legal vector type in LLT.
   if (cast<FixedVectorType>(U.getType())->getNumElements() == 1)
     return translateCopy(U, *U.getOperand(1), MIRBuilder);
 
   Register Res = getOrCreateVReg(U);
   Register Val = getOrCreateVReg(*U.getOperand(0));
   Register Elt = getOrCreateVReg(*U.getOperand(1));
   Register Idx = getOrCreateVReg(*U.getOperand(2));
   MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
   return true;
 }
 
 bool IRTranslator::translateExtractElement(const User &U,
                                            MachineIRBuilder &MIRBuilder) {
   // If it is a <1 x Ty> vector, use the scalar as it is
   // not a legal vector type in LLT.
   if (cast<FixedVectorType>(U.getOperand(0)->getType())->getNumElements() == 1)
     return translateCopy(U, *U.getOperand(0), MIRBuilder);
 
   Register Res = getOrCreateVReg(U);
   Register Val = getOrCreateVReg(*U.getOperand(0));
   const auto &TLI = *MF->getSubtarget().getTargetLowering();
   unsigned PreferredVecIdxWidth = TLI.getVectorIdxTy(*DL).getSizeInBits();
   Register Idx;
   if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {
     if (CI->getBitWidth() != PreferredVecIdxWidth) {
       APInt NewIdx = CI->getValue().sextOrTrunc(PreferredVecIdxWidth);
       auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
       Idx = getOrCreateVReg(*NewIdxCI);
     }
   }
   if (!Idx)
     Idx = getOrCreateVReg(*U.getOperand(1));
   if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
     const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
     Idx = MIRBuilder.buildSExtOrTrunc(VecIdxTy, Idx).getReg(0);
   }
   MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
   return true;
 }
 
 bool IRTranslator::translateShuffleVector(const User &U,
                                           MachineIRBuilder &MIRBuilder) {
   ArrayRef<int> Mask;
   if (auto *SVI = dyn_cast<ShuffleVectorInst>(&U))
     Mask = SVI->getShuffleMask();
   else
     Mask = cast<ConstantExpr>(U).getShuffleMask();
   ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);
   MIRBuilder
       .buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},
                   {getOrCreateVReg(*U.getOperand(0)),
                    getOrCreateVReg(*U.getOperand(1))})
       .addShuffleMask(MaskAlloc);
   return true;
 }
 
 bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
   const PHINode &PI = cast<PHINode>(U);
 
   SmallVector<MachineInstr *, 4> Insts;
   for (auto Reg : getOrCreateVRegs(PI)) {
     auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, {Reg}, {});
     Insts.push_back(MIB.getInstr());
   }
 
   PendingPHIs.emplace_back(&PI, std::move(Insts));
   return true;
 }
 
 bool IRTranslator::translateAtomicCmpXchg(const User &U,
                                           MachineIRBuilder &MIRBuilder) {
   const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
 
   auto &TLI = *MF->getSubtarget().getTargetLowering();
   auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
 
   auto Res = getOrCreateVRegs(I);
   Register OldValRes = Res[0];
   Register SuccessRes = Res[1];
   Register Addr = getOrCreateVReg(*I.getPointerOperand());
   Register Cmp = getOrCreateVReg(*I.getCompareOperand());
   Register NewVal = getOrCreateVReg(*I.getNewValOperand());
 
   MIRBuilder.buildAtomicCmpXchgWithSuccess(
       OldValRes, SuccessRes, Addr, Cmp, NewVal,
       *MF->getMachineMemOperand(
           MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp),
           getMemOpAlign(I), I.getAAMetadata(), nullptr, I.getSyncScopeID(),
           I.getSuccessOrdering(), I.getFailureOrdering()));
   return true;
 }
 
 bool IRTranslator::translateAtomicRMW(const User &U,
                                       MachineIRBuilder &MIRBuilder) {
   const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
   auto &TLI = *MF->getSubtarget().getTargetLowering();
   auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
 
   Register Res = getOrCreateVReg(I);
   Register Addr = getOrCreateVReg(*I.getPointerOperand());
   Register Val = getOrCreateVReg(*I.getValOperand());
 
   unsigned Opcode = 0;
   switch (I.getOperation()) {
   default:
     return false;
   case AtomicRMWInst::Xchg:
     Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
     break;
   case AtomicRMWInst::Add:
     Opcode = TargetOpcode::G_ATOMICRMW_ADD;
     break;
   case AtomicRMWInst::Sub:
     Opcode = TargetOpcode::G_ATOMICRMW_SUB;
     break;
   case AtomicRMWInst::And:
     Opcode = TargetOpcode::G_ATOMICRMW_AND;
     break;
   case AtomicRMWInst::Nand:
     Opcode = TargetOpcode::G_ATOMICRMW_NAND;
     break;
   case AtomicRMWInst::Or:
     Opcode = TargetOpcode::G_ATOMICRMW_OR;
     break;
   case AtomicRMWInst::Xor:
     Opcode = TargetOpcode::G_ATOMICRMW_XOR;
     break;
   case AtomicRMWInst::Max:
     Opcode = TargetOpcode::G_ATOMICRMW_MAX;
     break;
   case AtomicRMWInst::Min:
     Opcode = TargetOpcode::G_ATOMICRMW_MIN;
     break;
   case AtomicRMWInst::UMax:
     Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
     break;
   case AtomicRMWInst::UMin:
     Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
     break;
   case AtomicRMWInst::FAdd:
     Opcode = TargetOpcode::G_ATOMICRMW_FADD;
     break;
   case AtomicRMWInst::FSub:
     Opcode = TargetOpcode::G_ATOMICRMW_FSUB;
     break;
   }
 
   MIRBuilder.buildAtomicRMW(
       Opcode, Res, Addr, Val,
       *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
                                 Flags, MRI->getType(Val), getMemOpAlign(I),
                                 I.getAAMetadata(), nullptr, I.getSyncScopeID(),
                                 I.getOrdering()));
   return true;
 }
 
 bool IRTranslator::translateFence(const User &U,
                                   MachineIRBuilder &MIRBuilder) {
   const FenceInst &Fence = cast<FenceInst>(U);
   MIRBuilder.buildFence(static_cast<unsigned>(Fence.getOrdering()),
                         Fence.getSyncScopeID());
   return true;
 }
 
 bool IRTranslator::translateFreeze(const User &U,
                                    MachineIRBuilder &MIRBuilder) {
   const ArrayRef<Register> DstRegs = getOrCreateVRegs(U);
   const ArrayRef<Register> SrcRegs = getOrCreateVRegs(*U.getOperand(0));
 
   assert(DstRegs.size() == SrcRegs.size() &&
          "Freeze with different source and destination type?");
 
   for (unsigned I = 0; I < DstRegs.size(); ++I) {
     MIRBuilder.buildFreeze(DstRegs[I], SrcRegs[I]);
   }
 
   return true;
 }
 
 void IRTranslator::finishPendingPhis() {
 #ifndef NDEBUG
   DILocationVerifier Verifier;
   GISelObserverWrapper WrapperObserver(&Verifier);
   RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
 #endif // ifndef NDEBUG
   for (auto &Phi : PendingPHIs) {
     const PHINode *PI = Phi.first;
     ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
     MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
     EntryBuilder->setDebugLoc(PI->getDebugLoc());
 #ifndef NDEBUG
     Verifier.setCurrentInst(PI);
 #endif // ifndef NDEBUG
 
     SmallSet<const MachineBasicBlock *, 16> SeenPreds;
     for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
       auto IRPred = PI->getIncomingBlock(i);
       ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
       for (auto Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
         if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred))
           continue;
         SeenPreds.insert(Pred);
         for (unsigned j = 0; j < ValRegs.size(); ++j) {
           MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
           MIB.addUse(ValRegs[j]);
           MIB.addMBB(Pred);
         }
       }
     }
   }
 }
 
 bool IRTranslator::valueIsSplit(const Value &V,
                                 SmallVectorImpl<uint64_t> *Offsets) {
   SmallVector<LLT, 4> SplitTys;
   if (Offsets && !Offsets->empty())
     Offsets->clear();
   computeValueLLTs(*DL, *V.getType(), SplitTys, Offsets);
   return SplitTys.size() > 1;
 }
 
 bool IRTranslator::translate(const Instruction &Inst) {
   CurBuilder->setDebugLoc(Inst.getDebugLoc());
 
   auto &TLI = *MF->getSubtarget().getTargetLowering();
   if (TLI.fallBackToDAGISel(Inst))
     return false;
 
   switch (Inst.getOpcode()) {
 #define HANDLE_INST(NUM, OPCODE, CLASS)                                        \
   case Instruction::OPCODE:                                                    \
     return translate##OPCODE(Inst, *CurBuilder.get());
 #include "llvm/IR/Instruction.def"
   default:
     return false;
   }
 }
 
 bool IRTranslator::translate(const Constant &C, Register Reg) {
   // We only emit constants into the entry block from here. To prevent jumpy
   // debug behaviour set the line to 0.
   if (auto CurrInstDL = CurBuilder->getDL())
     EntryBuilder->setDebugLoc(DILocation::get(C.getContext(), 0, 0,
                                               CurrInstDL.getScope(),
                                               CurrInstDL.getInlinedAt()));
 
   if (auto CI = dyn_cast<ConstantInt>(&C))
     EntryBuilder->buildConstant(Reg, *CI);
   else if (auto CF = dyn_cast<ConstantFP>(&C))
     EntryBuilder->buildFConstant(Reg, *CF);
   else if (isa<UndefValue>(C))
     EntryBuilder->buildUndef(Reg);
   else if (isa<ConstantPointerNull>(C))
     EntryBuilder->buildConstant(Reg, 0);
   else if (auto GV = dyn_cast<GlobalValue>(&C))
     EntryBuilder->buildGlobalValue(Reg, GV);
   else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
     if (!isa<FixedVectorType>(CAZ->getType()))
       return false;
     // Return the scalar if it is a <1 x Ty> vector.
     unsigned NumElts = CAZ->getElementCount().getFixedValue();
     if (NumElts == 1)
       return translateCopy(C, *CAZ->getElementValue(0u), *EntryBuilder.get());
     SmallVector<Register, 4> Ops;
     for (unsigned I = 0; I < NumElts; ++I) {
       Constant &Elt = *CAZ->getElementValue(I);
       Ops.push_back(getOrCreateVReg(Elt));
     }
     EntryBuilder->buildBuildVector(Reg, Ops);
   } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
     // Return the scalar if it is a <1 x Ty> vector.
     if (CV->getNumElements() == 1)
       return translateCopy(C, *CV->getElementAsConstant(0),
                            *EntryBuilder.get());
     SmallVector<Register, 4> Ops;
     for (unsigned i = 0; i < CV->getNumElements(); ++i) {
       Constant &Elt = *CV->getElementAsConstant(i);
       Ops.push_back(getOrCreateVReg(Elt));
     }
     EntryBuilder->buildBuildVector(Reg, Ops);
   } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
     switch(CE->getOpcode()) {
 #define HANDLE_INST(NUM, OPCODE, CLASS)                                        \
   case Instruction::OPCODE:                                                    \
     return translate##OPCODE(*CE, *EntryBuilder.get());
 #include "llvm/IR/Instruction.def"
     default:
       return false;
     }
   } else if (auto CV = dyn_cast<ConstantVector>(&C)) {
     if (CV->getNumOperands() == 1)
       return translateCopy(C, *CV->getOperand(0), *EntryBuilder.get());
     SmallVector<Register, 4> Ops;
     for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
       Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
     }
     EntryBuilder->buildBuildVector(Reg, Ops);
   } else if (auto *BA = dyn_cast<BlockAddress>(&C)) {
     EntryBuilder->buildBlockAddress(Reg, BA);
   } else
     return false;
 
   return true;
 }
 
 bool IRTranslator::finalizeBasicBlock(const BasicBlock &BB,
                                       MachineBasicBlock &MBB) {
   for (auto &BTB : SL->BitTestCases) {
     // Emit header first, if it wasn't already emitted.
     if (!BTB.Emitted)
       emitBitTestHeader(BTB, BTB.Parent);
 
     BranchProbability UnhandledProb = BTB.Prob;
     for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
       UnhandledProb -= BTB.Cases[j].ExtraProb;
       // Set the current basic block to the mbb we wish to insert the code into
       MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;
       // If all cases cover a contiguous range, it is not necessary to jump to
       // the default block after the last bit test fails. This is because the
       // range check during bit test header creation has guaranteed that every
       // case here doesn't go outside the range. In this case, there is no need
       // to perform the last bit test, as it will always be true. Instead, make
       // the second-to-last bit-test fall through to the target of the last bit
       // test, and delete the last bit test.
 
       MachineBasicBlock *NextMBB;
       if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
         // Second-to-last bit-test with contiguous range: fall through to the
         // target of the final bit test.
         NextMBB = BTB.Cases[j + 1].TargetBB;
       } else if (j + 1 == ej) {
         // For the last bit test, fall through to Default.
         NextMBB = BTB.Default;
       } else {
         // Otherwise, fall through to the next bit test.
         NextMBB = BTB.Cases[j + 1].ThisBB;
       }
 
       emitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j], MBB);
 
       if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
         // We need to record the replacement phi edge here that normally
         // happens in emitBitTestCase before we delete the case, otherwise the
         // phi edge will be lost.
         addMachineCFGPred({BTB.Parent->getBasicBlock(),
                            BTB.Cases[ej - 1].TargetBB->getBasicBlock()},
                           MBB);
         // Since we're not going to use the final bit test, remove it.
         BTB.Cases.pop_back();
         break;
       }
     }
     // This is "default" BB. We have two jumps to it. From "header" BB and from
     // last "case" BB, unless the latter was skipped.
     CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),
                                    BTB.Default->getBasicBlock()};
     addMachineCFGPred(HeaderToDefaultEdge, BTB.Parent);
     if (!BTB.ContiguousRange) {
       addMachineCFGPred(HeaderToDefaultEdge, BTB.Cases.back().ThisBB);
     }
   }
   SL->BitTestCases.clear();
 
   for (auto &JTCase : SL->JTCases) {
     // Emit header first, if it wasn't already emitted.
     if (!JTCase.first.Emitted)
       emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);
 
     emitJumpTable(JTCase.second, JTCase.second.MBB);
   }
   SL->JTCases.clear();
 
   for (auto &SwCase : SL->SwitchCases)
     emitSwitchCase(SwCase, &CurBuilder->getMBB(), *CurBuilder);
   SL->SwitchCases.clear();
 
   // Check if we need to generate stack-protector guard checks.
   StackProtector &SP = getAnalysis<StackProtector>();
   if (SP.shouldEmitSDCheck(BB)) {
     const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
     bool FunctionBasedInstrumentation =
         TLI.getSSPStackGuardCheck(*MF->getFunction().getParent());
     SPDescriptor.initialize(&BB, &MBB, FunctionBasedInstrumentation);
   }
   // Handle stack protector.
   if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
     LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");
     return false;
   } else if (SPDescriptor.shouldEmitStackProtector()) {
     MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();
     MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();
 
     // Find the split point to split the parent mbb. At the same time copy all
     // physical registers used in the tail of parent mbb into virtual registers
     // before the split point and back into physical registers after the split
     // point. This prevents us needing to deal with Live-ins and many other
     // register allocation issues caused by us splitting the parent mbb. The
     // register allocator will clean up said virtual copies later on.
     MachineBasicBlock::iterator SplitPoint = findSplitPointForStackProtector(
         ParentMBB, *MF->getSubtarget().getInstrInfo());
 
     // Splice the terminator of ParentMBB into SuccessMBB.
     SuccessMBB->splice(SuccessMBB->end(), ParentMBB, SplitPoint,
                        ParentMBB->end());
 
     // Add compare/jump on neq/jump to the parent BB.
     if (!emitSPDescriptorParent(SPDescriptor, ParentMBB))
       return false;
 
     // CodeGen Failure MBB if we have not codegened it yet.
     MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();
     if (FailureMBB->empty()) {
       if (!emitSPDescriptorFailure(SPDescriptor, FailureMBB))
         return false;
     }
 
     // Clear the Per-BB State.
     SPDescriptor.resetPerBBState();
   }
   return true;
 }
 
 bool IRTranslator::emitSPDescriptorParent(StackProtectorDescriptor &SPD,
                                           MachineBasicBlock *ParentBB) {
   CurBuilder->setInsertPt(*ParentBB, ParentBB->end());
   // First create the loads to the guard/stack slot for the comparison.
   const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
   Type *PtrIRTy = Type::getInt8PtrTy(MF->getFunction().getContext());
   const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
   LLT PtrMemTy = getLLTForMVT(TLI.getPointerMemTy(*DL));
 
   MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
   int FI = MFI.getStackProtectorIndex();
 
   Register Guard;
   Register StackSlotPtr = CurBuilder->buildFrameIndex(PtrTy, FI).getReg(0);
   const Module &M = *ParentBB->getParent()->getFunction().getParent();
   Align Align = DL->getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));
 
   // Generate code to load the content of the guard slot.
   Register GuardVal =
       CurBuilder
           ->buildLoad(PtrMemTy, StackSlotPtr,
                       MachinePointerInfo::getFixedStack(*MF, FI), Align,
                       MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile)
           .getReg(0);
 
   if (TLI.useStackGuardXorFP()) {
     LLVM_DEBUG(dbgs() << "Stack protector xor'ing with FP not yet implemented");
     return false;
   }
 
   // Retrieve guard check function, nullptr if instrumentation is inlined.
   if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
     // This path is currently untestable on GlobalISel, since the only platform
     // that needs this seems to be Windows, and we fall back on that currently.
     // The code still lives here in case that changes.
     // Silence warning about unused variable until the code below that uses
     // 'GuardCheckFn' is enabled.
     (void)GuardCheckFn;
     return false;
 #if 0
     // The target provides a guard check function to validate the guard value.
     // Generate a call to that function with the content of the guard slot as
     // argument.
     FunctionType *FnTy = GuardCheckFn->getFunctionType();
     assert(FnTy->getNumParams() == 1 && "Invalid function signature");
     ISD::ArgFlagsTy Flags;
     if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
       Flags.setInReg();
     CallLowering::ArgInfo GuardArgInfo(
         {GuardVal, FnTy->getParamType(0), {Flags}});
 
     CallLowering::CallLoweringInfo Info;
     Info.OrigArgs.push_back(GuardArgInfo);
     Info.CallConv = GuardCheckFn->getCallingConv();
     Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);
     Info.OrigRet = {Register(), FnTy->getReturnType()};
     if (!CLI->lowerCall(MIRBuilder, Info)) {
       LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");
       return false;
     }
     return true;
 #endif
   }
 
   // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
   // Otherwise, emit a volatile load to retrieve the stack guard value.
   if (TLI.useLoadStackGuardNode()) {
     Guard =
         MRI->createGenericVirtualRegister(LLT::scalar(PtrTy.getSizeInBits()));
     getStackGuard(Guard, *CurBuilder);
   } else {
     // TODO: test using android subtarget when we support @llvm.thread.pointer.
     const Value *IRGuard = TLI.getSDagStackGuard(M);
     Register GuardPtr = getOrCreateVReg(*IRGuard);
 
     Guard = CurBuilder
                 ->buildLoad(PtrMemTy, GuardPtr,
                             MachinePointerInfo::getFixedStack(*MF, FI), Align,
                             MachineMemOperand::MOLoad |
                                 MachineMemOperand::MOVolatile)
                 .getReg(0);
   }
 
   // Perform the comparison.
   auto Cmp =
       CurBuilder->buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Guard, GuardVal);
   // If the guard/stackslot do not equal, branch to failure MBB.
   CurBuilder->buildBrCond(Cmp, *SPD.getFailureMBB());
   // Otherwise branch to success MBB.
   CurBuilder->buildBr(*SPD.getSuccessMBB());
   return true;
 }
 
 bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,
                                            MachineBasicBlock *FailureBB) {
   CurBuilder->setInsertPt(*FailureBB, FailureBB->end());
   const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
 
   const RTLIB::Libcall Libcall = RTLIB::STACKPROTECTOR_CHECK_FAIL;
   const char *Name = TLI.getLibcallName(Libcall);
 
   CallLowering::CallLoweringInfo Info;
   Info.CallConv = TLI.getLibcallCallingConv(Libcall);
   Info.Callee = MachineOperand::CreateES(Name);
   Info.OrigRet = {Register(), Type::getVoidTy(MF->getFunction().getContext()),
                   0};
   if (!CLI->lowerCall(*CurBuilder, Info)) {
     LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");
     return false;
   }
 
   // On PS4, the "return address" must still be within the calling function,
   // even if it's at the very end, so emit an explicit TRAP here.
   // Passing 'true' for doesNotReturn above won't generate the trap for us.
   // WebAssembly needs an unreachable instruction after a non-returning call,
   // because the function return type can be different from __stack_chk_fail's
   // return type (void).
   const TargetMachine &TM = MF->getTarget();
   if (TM.getTargetTriple().isPS4CPU() || TM.getTargetTriple().isWasm()) {
     LLVM_DEBUG(dbgs() << "Unhandled trap emission for stack protector fail\n");
     return false;
   }
   return true;
 }
 
 void IRTranslator::finalizeFunction() {
   // Release the memory used by the different maps we
   // needed during the translation.
   PendingPHIs.clear();
   VMap.reset();
   FrameIndices.clear();
   MachinePreds.clear();
   // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
   // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
   // destroying it twice (in ~IRTranslator() and ~LLVMContext())
   EntryBuilder.reset();
   CurBuilder.reset();
   FuncInfo.clear();
   SPDescriptor.resetPerFunctionState();
 }
 
 /// Returns true if a BasicBlock \p BB within a variadic function contains a
 /// variadic musttail call.
 static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
   if (!IsVarArg)
     return false;
 
   // Walk the block backwards, because tail calls usually only appear at the end
   // of a block.
   return llvm::any_of(llvm::reverse(BB), [](const Instruction &I) {
     const auto *CI = dyn_cast<CallInst>(&I);
     return CI && CI->isMustTailCall();
   });
 }
 
 bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
   MF = &CurMF;
   const Function &F = MF->getFunction();
   GISelCSEAnalysisWrapper &Wrapper =
       getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
   // Set the CSEConfig and run the analysis.
   GISelCSEInfo *CSEInfo = nullptr;
   TPC = &getAnalysis<TargetPassConfig>();
   bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
                        ? EnableCSEInIRTranslator
                        : TPC->isGISelCSEEnabled();
 
   if (EnableCSE) {
     EntryBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
     CSEInfo = &Wrapper.get(TPC->getCSEConfig());
     EntryBuilder->setCSEInfo(CSEInfo);
     CurBuilder = std::make_unique<CSEMIRBuilder>(CurMF);
     CurBuilder->setCSEInfo(CSEInfo);
   } else {
     EntryBuilder = std::make_unique<MachineIRBuilder>();
     CurBuilder = std::make_unique<MachineIRBuilder>();
   }
   CLI = MF->getSubtarget().getCallLowering();
   CurBuilder->setMF(*MF);
   EntryBuilder->setMF(*MF);
   MRI = &MF->getRegInfo();
   DL = &F.getParent()->getDataLayout();
   ORE = std::make_unique<OptimizationRemarkEmitter>(&F);
   const TargetMachine &TM = MF->getTarget();
   TM.resetTargetOptions(F);
   EnableOpts = OptLevel != CodeGenOpt::None && !skipFunction(F);
   FuncInfo.MF = MF;
   if (EnableOpts)
     FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
   else
     FuncInfo.BPI = nullptr;
 
   FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(*MF);
 
   const auto &TLI = *MF->getSubtarget().getTargetLowering();
 
   SL = std::make_unique<GISelSwitchLowering>(this, FuncInfo);
   SL->init(TLI, TM, *DL);
 
 
 
   assert(PendingPHIs.empty() && "stale PHIs");
 
   // Targets which want to use big endian can enable it using
   // enableBigEndian()
   if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
     // Currently we don't properly handle big endian code.
     OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
                                F.getSubprogram(), &F.getEntryBlock());
     R << "unable to translate in big endian mode";
     reportTranslationError(*MF, *TPC, *ORE, R);
   }
 
   // Release the per-function state when we return, whether we succeeded or not.
   auto FinalizeOnReturn = make_scope_exit([this]() { finalizeFunction(); });
 
   // Setup a separate basic-block for the arguments and constants
   MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
   MF->push_back(EntryBB);
   EntryBuilder->setMBB(*EntryBB);
 
   DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHI()->getDebugLoc();
   SwiftError.setFunction(CurMF);
   SwiftError.createEntriesInEntryBlock(DbgLoc);
 
   bool IsVarArg = F.isVarArg();
   bool HasMustTailInVarArgFn = false;
 
   // Create all blocks, in IR order, to preserve the layout.
   for (const BasicBlock &BB: F) {
     auto *&MBB = BBToMBB[&BB];
 
     MBB = MF->CreateMachineBasicBlock(&BB);
     MF->push_back(MBB);
 
     if (BB.hasAddressTaken())
       MBB->setHasAddressTaken();
 
     if (!HasMustTailInVarArgFn)
       HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
   }
 
   MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
 
   // Make our arguments/constants entry block fallthrough to the IR entry block.
   EntryBB->addSuccessor(&getMBB(F.front()));
 
   if (CLI->fallBackToDAGISel(*MF)) {
     OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
                                F.getSubprogram(), &F.getEntryBlock());
     R << "unable to lower function: " << ore::NV("Prototype", F.getType());
     reportTranslationError(*MF, *TPC, *ORE, R);
     return false;
   }
 
   // Lower the actual args into this basic block.
   SmallVector<ArrayRef<Register>, 8> VRegArgs;
   for (const Argument &Arg: F.args()) {
     if (DL->getTypeStoreSize(Arg.getType()).isZero())
       continue; // Don't handle zero sized types.
     ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);
     VRegArgs.push_back(VRegs);
 
     if (Arg.hasSwiftErrorAttr()) {
       assert(VRegs.size() == 1 && "Too many vregs for Swift error");
       SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
     }
   }
 
   if (!CLI->lowerFormalArguments(*EntryBuilder.get(), F, VRegArgs, FuncInfo)) {
     OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
                                F.getSubprogram(), &F.getEntryBlock());
     R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
     reportTranslationError(*MF, *TPC, *ORE, R);
     return false;
   }
 
   // Need to visit defs before uses when translating instructions.
   GISelObserverWrapper WrapperObserver;
   if (EnableCSE && CSEInfo)
     WrapperObserver.addObserver(CSEInfo);
   {
     ReversePostOrderTraversal<const Function *> RPOT(&F);
 #ifndef NDEBUG
     DILocationVerifier Verifier;
     WrapperObserver.addObserver(&Verifier);
 #endif // ifndef NDEBUG
     RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
     RAIIMFObserverInstaller ObsInstall(*MF, WrapperObserver);
     for (const BasicBlock *BB : RPOT) {
       MachineBasicBlock &MBB = getMBB(*BB);
       // Set the insertion point of all the following translations to
       // the end of this basic block.
       CurBuilder->setMBB(MBB);
       HasTailCall = false;
       for (const Instruction &Inst : *BB) {
         // If we translated a tail call in the last step, then we know
         // everything after the call is either a return, or something that is
         // handled by the call itself. (E.g. a lifetime marker or assume
         // intrinsic.) In this case, we should stop translating the block and
         // move on.
         if (HasTailCall)
           break;
 #ifndef NDEBUG
         Verifier.setCurrentInst(&Inst);
 #endif // ifndef NDEBUG
         if (translate(Inst))
           continue;
 
         OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
                                    Inst.getDebugLoc(), BB);
         R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
 
         if (ORE->allowExtraAnalysis("gisel-irtranslator")) {
           std::string InstStrStorage;
           raw_string_ostream InstStr(InstStrStorage);
           InstStr << Inst;
 
           R << ": '" << InstStr.str() << "'";
         }
 
         reportTranslationError(*MF, *TPC, *ORE, R);
         return false;
       }
 
-      if (!finalizeBasicBlock(*BB, MBB))
+      if (!finalizeBasicBlock(*BB, MBB)) {
+        OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
+                                   BB->getTerminator()->getDebugLoc(), BB);
+        R << "unable to translate basic block";
+        reportTranslationError(*MF, *TPC, *ORE, R);
         return false;
+      }
     }
 #ifndef NDEBUG
     WrapperObserver.removeObserver(&Verifier);
 #endif
   }
 
   finishPendingPhis();
 
   SwiftError.propagateVRegs();
 
   // Merge the argument lowering and constants block with its single
   // successor, the LLVM-IR entry block.  We want the basic block to
   // be maximal.
   assert(EntryBB->succ_size() == 1 &&
          "Custom BB used for lowering should have only one successor");
   // Get the successor of the current entry block.
   MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
   assert(NewEntryBB.pred_size() == 1 &&
          "LLVM-IR entry block has a predecessor!?");
   // Move all the instruction from the current entry block to the
   // new entry block.
   NewEntryBB.splice(NewEntryBB.begin(), EntryBB, EntryBB->begin(),
                     EntryBB->end());
 
   // Update the live-in information for the new entry block.
   for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
     NewEntryBB.addLiveIn(LiveIn);
   NewEntryBB.sortUniqueLiveIns();
 
   // Get rid of the now empty basic block.
   EntryBB->removeSuccessor(&NewEntryBB);
   MF->remove(EntryBB);
   MF->deleteMachineBasicBlock(EntryBB);
 
   assert(&MF->front() == &NewEntryBB &&
          "New entry wasn't next in the list of basic block!");
 
   // Initialize stack protector information.
   StackProtector &SP = getAnalysis<StackProtector>();
   SP.copyToMachineFrameInfo(MF->getFrameInfo());
 
   return false;
 }
diff --git a/contrib/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp b/contrib/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp
index 6fd2f7e7a718..e6d28141dd06 100644
--- a/contrib/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/contrib/llvm-project/llvm/lib/MC/ELFObjectWriter.cpp
@@ -1,1526 +1,1544 @@
 //===- lib/MC/ELFObjectWriter.cpp - ELF File Writer -----------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements ELF object file writer information.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCAsmLayout.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
 #include "llvm/MC/MCFixupKindInfo.h"
 #include "llvm/MC/MCFragment.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/MC/StringTableBuilder.h"
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/EndianStream.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/LEB128.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/SMLoc.h"
 #include "llvm/Support/StringSaver.h"
 #include "llvm/Support/SwapByteOrder.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
 #include <map>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
 using namespace llvm;
 
 #undef  DEBUG_TYPE
 #define DEBUG_TYPE "reloc-info"
 
 namespace {
 
 using SectionIndexMapTy = DenseMap<const MCSectionELF *, uint32_t>;
 
 class ELFObjectWriter;
 struct ELFWriter;
 
 bool isDwoSection(const MCSectionELF &Sec) {
   return Sec.getName().endswith(".dwo");
 }
 
 class SymbolTableWriter {
   ELFWriter &EWriter;
   bool Is64Bit;
 
   // indexes we are going to write to .symtab_shndx.
   std::vector<uint32_t> ShndxIndexes;
 
   // The numbel of symbols written so far.
   unsigned NumWritten;
 
   void createSymtabShndx();
 
   template <typename T> void write(T Value);
 
 public:
   SymbolTableWriter(ELFWriter &EWriter, bool Is64Bit);
 
   void writeSymbol(uint32_t name, uint8_t info, uint64_t value, uint64_t size,
                    uint8_t other, uint32_t shndx, bool Reserved);
 
   ArrayRef<uint32_t> getShndxIndexes() const { return ShndxIndexes; }
 };
 
 struct ELFWriter {
   ELFObjectWriter &OWriter;
   support::endian::Writer W;
 
   enum DwoMode {
     AllSections,
     NonDwoOnly,
     DwoOnly,
   } Mode;
 
   static uint64_t SymbolValue(const MCSymbol &Sym, const MCAsmLayout &Layout);
   static bool isInSymtab(const MCAsmLayout &Layout, const MCSymbolELF &Symbol,
                          bool Used, bool Renamed);
 
   /// Helper struct for containing some precomputed information on symbols.
   struct ELFSymbolData {
     const MCSymbolELF *Symbol;
     StringRef Name;
     uint32_t SectionIndex;
     uint32_t Order;
   };
 
   /// @}
   /// @name Symbol Table Data
   /// @{
 
   StringTableBuilder StrTabBuilder{StringTableBuilder::ELF};
 
   /// @}
 
   // This holds the symbol table index of the last local symbol.
   unsigned LastLocalSymbolIndex;
   // This holds the .strtab section index.
   unsigned StringTableIndex;
   // This holds the .symtab section index.
   unsigned SymbolTableIndex;
 
   // Sections in the order they are to be output in the section table.
   std::vector<const MCSectionELF *> SectionTable;
   unsigned addToSectionTable(const MCSectionELF *Sec);
 
   // TargetObjectWriter wrappers.
   bool is64Bit() const;
   bool usesRela(const MCSectionELF &Sec) const;
 
   uint64_t align(unsigned Alignment);
 
   bool maybeWriteCompression(uint64_t Size,
                              SmallVectorImpl<char> &CompressedContents,
                              bool ZLibStyle, unsigned Alignment);
 
 public:
   ELFWriter(ELFObjectWriter &OWriter, raw_pwrite_stream &OS,
             bool IsLittleEndian, DwoMode Mode)
       : OWriter(OWriter),
         W(OS, IsLittleEndian ? support::little : support::big), Mode(Mode) {}
 
   void WriteWord(uint64_t Word) {
     if (is64Bit())
       W.write<uint64_t>(Word);
     else
       W.write<uint32_t>(Word);
   }
 
   template <typename T> void write(T Val) {
     W.write(Val);
   }
 
   void writeHeader(const MCAssembler &Asm);
 
   void writeSymbol(SymbolTableWriter &Writer, uint32_t StringIndex,
                    ELFSymbolData &MSD, const MCAsmLayout &Layout);
 
   // Start and end offset of each section
   using SectionOffsetsTy =
       std::map<const MCSectionELF *, std::pair<uint64_t, uint64_t>>;
 
   // Map from a signature symbol to the group section index
   using RevGroupMapTy = DenseMap<const MCSymbol *, unsigned>;
 
   /// Compute the symbol table data
   ///
   /// \param Asm - The assembler.
   /// \param SectionIndexMap - Maps a section to its index.
   /// \param RevGroupMap - Maps a signature symbol to the group section.
   void computeSymbolTable(MCAssembler &Asm, const MCAsmLayout &Layout,
                           const SectionIndexMapTy &SectionIndexMap,
                           const RevGroupMapTy &RevGroupMap,
                           SectionOffsetsTy &SectionOffsets);
 
   void writeAddrsigSection();
 
   MCSectionELF *createRelocationSection(MCContext &Ctx,
                                         const MCSectionELF &Sec);
 
   void writeSectionHeader(const MCAsmLayout &Layout,
                           const SectionIndexMapTy &SectionIndexMap,
                           const SectionOffsetsTy &SectionOffsets);
 
   void writeSectionData(const MCAssembler &Asm, MCSection &Sec,
                         const MCAsmLayout &Layout);
 
   void WriteSecHdrEntry(uint32_t Name, uint32_t Type, uint64_t Flags,
                         uint64_t Address, uint64_t Offset, uint64_t Size,
                         uint32_t Link, uint32_t Info, uint64_t Alignment,
                         uint64_t EntrySize);
 
   void writeRelocations(const MCAssembler &Asm, const MCSectionELF &Sec);
 
   uint64_t writeObject(MCAssembler &Asm, const MCAsmLayout &Layout);
   void writeSection(const SectionIndexMapTy &SectionIndexMap,
                     uint32_t GroupSymbolIndex, uint64_t Offset, uint64_t Size,
                     const MCSectionELF &Section);
 };
 
 class ELFObjectWriter : public MCObjectWriter {
   /// The target specific ELF writer instance.
   std::unique_ptr<MCELFObjectTargetWriter> TargetObjectWriter;
 
   DenseMap<const MCSectionELF *, std::vector<ELFRelocationEntry>> Relocations;
 
   DenseMap<const MCSymbolELF *, const MCSymbolELF *> Renames;
 
   bool SeenGnuAbi = false;
   bool EmitAddrsigSection = false;
   std::vector<const MCSymbol *> AddrsigSyms;
 
   bool hasRelocationAddend() const;
 
   bool shouldRelocateWithSymbol(const MCAssembler &Asm,
                                 const MCSymbolRefExpr *RefA,
                                 const MCSymbolELF *Sym, uint64_t C,
                                 unsigned Type) const;
 
 public:
   ELFObjectWriter(std::unique_ptr<MCELFObjectTargetWriter> MOTW)
       : TargetObjectWriter(std::move(MOTW)) {}
 
   void reset() override {
     SeenGnuAbi = false;
     Relocations.clear();
     Renames.clear();
     MCObjectWriter::reset();
   }
 
   bool isSymbolRefDifferenceFullyResolvedImpl(const MCAssembler &Asm,
                                               const MCSymbol &SymA,
                                               const MCFragment &FB, bool InSet,
                                               bool IsPCRel) const override;
 
   virtual bool checkRelocation(MCContext &Ctx, SMLoc Loc,
                                const MCSectionELF *From,
                                const MCSectionELF *To) {
     return true;
   }
 
   void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout,
                         const MCFragment *Fragment, const MCFixup &Fixup,
                         MCValue Target, uint64_t &FixedValue) override;
 
   void executePostLayoutBinding(MCAssembler &Asm,
                                 const MCAsmLayout &Layout) override;
 
   void markGnuAbi() override { SeenGnuAbi = true; }
   bool seenGnuAbi() const { return SeenGnuAbi; }
   void emitAddrsigSection() override { EmitAddrsigSection = true; }
   void addAddrsigSymbol(const MCSymbol *Sym) override {
     AddrsigSyms.push_back(Sym);
   }
 
   friend struct ELFWriter;
 };
 
 class ELFSingleObjectWriter : public ELFObjectWriter {
   raw_pwrite_stream &OS;
   bool IsLittleEndian;
 
 public:
   ELFSingleObjectWriter(std::unique_ptr<MCELFObjectTargetWriter> MOTW,
                         raw_pwrite_stream &OS, bool IsLittleEndian)
       : ELFObjectWriter(std::move(MOTW)), OS(OS),
         IsLittleEndian(IsLittleEndian) {}
 
   uint64_t writeObject(MCAssembler &Asm, const MCAsmLayout &Layout) override {
     return ELFWriter(*this, OS, IsLittleEndian, ELFWriter::AllSections)
         .writeObject(Asm, Layout);
   }
 
   friend struct ELFWriter;
 };
 
 class ELFDwoObjectWriter : public ELFObjectWriter {
   raw_pwrite_stream &OS, &DwoOS;
   bool IsLittleEndian;
 
 public:
   ELFDwoObjectWriter(std::unique_ptr<MCELFObjectTargetWriter> MOTW,
                      raw_pwrite_stream &OS, raw_pwrite_stream &DwoOS,
                      bool IsLittleEndian)
       : ELFObjectWriter(std::move(MOTW)), OS(OS), DwoOS(DwoOS),
         IsLittleEndian(IsLittleEndian) {}
 
   virtual bool checkRelocation(MCContext &Ctx, SMLoc Loc,
                                const MCSectionELF *From,
                                const MCSectionELF *To) override {
     if (isDwoSection(*From)) {
       Ctx.reportError(Loc, "A dwo section may not contain relocations");
       return false;
     }
     if (To && isDwoSection(*To)) {
       Ctx.reportError(Loc, "A relocation may not refer to a dwo section");
       return false;
     }
     return true;
   }
 
   uint64_t writeObject(MCAssembler &Asm, const MCAsmLayout &Layout) override {
     uint64_t Size = ELFWriter(*this, OS, IsLittleEndian, ELFWriter::NonDwoOnly)
                         .writeObject(Asm, Layout);
     Size += ELFWriter(*this, DwoOS, IsLittleEndian, ELFWriter::DwoOnly)
                 .writeObject(Asm, Layout);
     return Size;
   }
 };
 
 } // end anonymous namespace
 
 uint64_t ELFWriter::align(unsigned Alignment) {
   uint64_t Offset = W.OS.tell(), NewOffset = alignTo(Offset, Alignment);
   W.OS.write_zeros(NewOffset - Offset);
   return NewOffset;
 }
 
 unsigned ELFWriter::addToSectionTable(const MCSectionELF *Sec) {
   SectionTable.push_back(Sec);
   StrTabBuilder.add(Sec->getName());
   return SectionTable.size();
 }
 
 void SymbolTableWriter::createSymtabShndx() {
   if (!ShndxIndexes.empty())
     return;
 
   ShndxIndexes.resize(NumWritten);
 }
 
 template <typename T> void SymbolTableWriter::write(T Value) {
   EWriter.write(Value);
 }
 
 SymbolTableWriter::SymbolTableWriter(ELFWriter &EWriter, bool Is64Bit)
     : EWriter(EWriter), Is64Bit(Is64Bit), NumWritten(0) {}
 
 void SymbolTableWriter::writeSymbol(uint32_t name, uint8_t info, uint64_t value,
                                     uint64_t size, uint8_t other,
                                     uint32_t shndx, bool Reserved) {
   bool LargeIndex = shndx >= ELF::SHN_LORESERVE && !Reserved;
 
   if (LargeIndex)
     createSymtabShndx();
 
   if (!ShndxIndexes.empty()) {
     if (LargeIndex)
       ShndxIndexes.push_back(shndx);
     else
       ShndxIndexes.push_back(0);
   }
 
   uint16_t Index = LargeIndex ? uint16_t(ELF::SHN_XINDEX) : shndx;
 
   if (Is64Bit) {
     write(name);  // st_name
     write(info);  // st_info
     write(other); // st_other
     write(Index); // st_shndx
     write(value); // st_value
     write(size);  // st_size
   } else {
     write(name);            // st_name
     write(uint32_t(value)); // st_value
     write(uint32_t(size));  // st_size
     write(info);            // st_info
     write(other);           // st_other
     write(Index);           // st_shndx
   }
 
   ++NumWritten;
 }
 
 bool ELFWriter::is64Bit() const {
   return OWriter.TargetObjectWriter->is64Bit();
 }
 
 bool ELFWriter::usesRela(const MCSectionELF &Sec) const {
   return OWriter.hasRelocationAddend() &&
          Sec.getType() != ELF::SHT_LLVM_CALL_GRAPH_PROFILE;
 }
 
 // Emit the ELF header.
 void ELFWriter::writeHeader(const MCAssembler &Asm) {
   // ELF Header
   // ----------
   //
   // Note
   // ----
   // emitWord method behaves differently for ELF32 and ELF64, writing
   // 4 bytes in the former and 8 in the latter.
 
   W.OS << ELF::ElfMagic; // e_ident[EI_MAG0] to e_ident[EI_MAG3]
 
   W.OS << char(is64Bit() ? ELF::ELFCLASS64 : ELF::ELFCLASS32); // e_ident[EI_CLASS]
 
   // e_ident[EI_DATA]
   W.OS << char(W.Endian == support::little ? ELF::ELFDATA2LSB
                                            : ELF::ELFDATA2MSB);
 
   W.OS << char(ELF::EV_CURRENT);        // e_ident[EI_VERSION]
   // e_ident[EI_OSABI]
   uint8_t OSABI = OWriter.TargetObjectWriter->getOSABI();
   W.OS << char(OSABI == ELF::ELFOSABI_NONE && OWriter.seenGnuAbi()
                    ? int(ELF::ELFOSABI_GNU)
                    : OSABI);
   // e_ident[EI_ABIVERSION]
   W.OS << char(OWriter.TargetObjectWriter->getABIVersion());
 
   W.OS.write_zeros(ELF::EI_NIDENT - ELF::EI_PAD);
 
   W.write<uint16_t>(ELF::ET_REL);             // e_type
 
   W.write<uint16_t>(OWriter.TargetObjectWriter->getEMachine()); // e_machine = target
 
   W.write<uint32_t>(ELF::EV_CURRENT);         // e_version
   WriteWord(0);                    // e_entry, no entry point in .o file
   WriteWord(0);                    // e_phoff, no program header for .o
   WriteWord(0);                     // e_shoff = sec hdr table off in bytes
 
   // e_flags = whatever the target wants
   W.write<uint32_t>(Asm.getELFHeaderEFlags());
 
   // e_ehsize = ELF header size
   W.write<uint16_t>(is64Bit() ? sizeof(ELF::Elf64_Ehdr)
                               : sizeof(ELF::Elf32_Ehdr));
 
   W.write<uint16_t>(0);                  // e_phentsize = prog header entry size
   W.write<uint16_t>(0);                  // e_phnum = # prog header entries = 0
 
   // e_shentsize = Section header entry size
   W.write<uint16_t>(is64Bit() ? sizeof(ELF::Elf64_Shdr)
                               : sizeof(ELF::Elf32_Shdr));
 
   // e_shnum     = # of section header ents
   W.write<uint16_t>(0);
 
   // e_shstrndx  = Section # of '.strtab'
   assert(StringTableIndex < ELF::SHN_LORESERVE);
   W.write<uint16_t>(StringTableIndex);
 }
 
 uint64_t ELFWriter::SymbolValue(const MCSymbol &Sym,
                                 const MCAsmLayout &Layout) {
   if (Sym.isCommon())
     return Sym.getCommonAlignment();
 
   uint64_t Res;
   if (!Layout.getSymbolOffset(Sym, Res))
     return 0;
 
   if (Layout.getAssembler().isThumbFunc(&Sym))
     Res |= 1;
 
   return Res;
 }
 
 static uint8_t mergeTypeForSet(uint8_t origType, uint8_t newType) {
   uint8_t Type = newType;
 
   // Propagation rules:
   // IFUNC > FUNC > OBJECT > NOTYPE
   // TLS_OBJECT > OBJECT > NOTYPE
   //
   // dont let the new type degrade the old type
   switch (origType) {
   default:
     break;
   case ELF::STT_GNU_IFUNC:
     if (Type == ELF::STT_FUNC || Type == ELF::STT_OBJECT ||
         Type == ELF::STT_NOTYPE || Type == ELF::STT_TLS)
       Type = ELF::STT_GNU_IFUNC;
     break;
   case ELF::STT_FUNC:
     if (Type == ELF::STT_OBJECT || Type == ELF::STT_NOTYPE ||
         Type == ELF::STT_TLS)
       Type = ELF::STT_FUNC;
     break;
   case ELF::STT_OBJECT:
     if (Type == ELF::STT_NOTYPE)
       Type = ELF::STT_OBJECT;
     break;
   case ELF::STT_TLS:
     if (Type == ELF::STT_OBJECT || Type == ELF::STT_NOTYPE ||
         Type == ELF::STT_GNU_IFUNC || Type == ELF::STT_FUNC)
       Type = ELF::STT_TLS;
     break;
   }
 
   return Type;
 }
 
 static bool isIFunc(const MCSymbolELF *Symbol) {
   while (Symbol->getType() != ELF::STT_GNU_IFUNC) {
     const MCSymbolRefExpr *Value;
     if (!Symbol->isVariable() ||
         !(Value = dyn_cast<MCSymbolRefExpr>(Symbol->getVariableValue())) ||
         Value->getKind() != MCSymbolRefExpr::VK_None ||
         mergeTypeForSet(Symbol->getType(), ELF::STT_GNU_IFUNC) != ELF::STT_GNU_IFUNC)
       return false;
     Symbol = &cast<MCSymbolELF>(Value->getSymbol());
   }
   return true;
 }
 
 void ELFWriter::writeSymbol(SymbolTableWriter &Writer, uint32_t StringIndex,
                             ELFSymbolData &MSD, const MCAsmLayout &Layout) {
   const auto &Symbol = cast<MCSymbolELF>(*MSD.Symbol);
   const MCSymbolELF *Base =
       cast_or_null<MCSymbolELF>(Layout.getBaseSymbol(Symbol));
 
   // This has to be in sync with when computeSymbolTable uses SHN_ABS or
   // SHN_COMMON.
   bool IsReserved = !Base || Symbol.isCommon();
 
   // Binding and Type share the same byte as upper and lower nibbles
   uint8_t Binding = Symbol.getBinding();
   uint8_t Type = Symbol.getType();
   if (isIFunc(&Symbol))
     Type = ELF::STT_GNU_IFUNC;
   if (Base) {
     Type = mergeTypeForSet(Type, Base->getType());
   }
   uint8_t Info = (Binding << 4) | Type;
 
   // Other and Visibility share the same byte with Visibility using the lower
   // 2 bits
   uint8_t Visibility = Symbol.getVisibility();
   uint8_t Other = Symbol.getOther() | Visibility;
 
   uint64_t Value = SymbolValue(*MSD.Symbol, Layout);
   uint64_t Size = 0;
 
   const MCExpr *ESize = MSD.Symbol->getSize();
-  if (!ESize && Base)
+  if (!ESize && Base) {
+    // For expressions like .set y, x+1, if y's size is unset, inherit from x.
     ESize = Base->getSize();
 
+    // For `.size x, 2; y = x; .size y, 1; z = y; z1 = z; .symver y, y@v1`, z,
+    // z1, and y@v1's st_size equals y's. However, `Base` is `x` which will give
+    // us 2. Follow the MCSymbolRefExpr assignment chain, which covers most
+    // needs. MCBinaryExpr is not handled.
+    const MCSymbolELF *Sym = &Symbol;
+    while (Sym->isVariable()) {
+      if (auto *Expr =
+              dyn_cast<MCSymbolRefExpr>(Sym->getVariableValue(false))) {
+        Sym = cast<MCSymbolELF>(&Expr->getSymbol());
+        if (!Sym->getSize())
+          continue;
+        ESize = Sym->getSize();
+      }
+      break;
+    }
+  }
+
   if (ESize) {
     int64_t Res;
     if (!ESize->evaluateKnownAbsolute(Res, Layout))
       report_fatal_error("Size expression must be absolute.");
     Size = Res;
   }
 
   // Write out the symbol table entry
   Writer.writeSymbol(StringIndex, Info, Value, Size, Other, MSD.SectionIndex,
                      IsReserved);
 }
 
 bool ELFWriter::isInSymtab(const MCAsmLayout &Layout, const MCSymbolELF &Symbol,
                            bool Used, bool Renamed) {
   if (Symbol.isVariable()) {
     const MCExpr *Expr = Symbol.getVariableValue();
     // Target Expressions that are always inlined do not appear in the symtab
     if (const auto *T = dyn_cast<MCTargetExpr>(Expr))
       if (T->inlineAssignedExpr())
         return false;
     if (const MCSymbolRefExpr *Ref = dyn_cast<MCSymbolRefExpr>(Expr)) {
       if (Ref->getKind() == MCSymbolRefExpr::VK_WEAKREF)
         return false;
     }
   }
 
   if (Used)
     return true;
 
   if (Renamed)
     return false;
 
   if (Symbol.isVariable() && Symbol.isUndefined()) {
     // FIXME: this is here just to diagnose the case of a var = commmon_sym.
     Layout.getBaseSymbol(Symbol);
     return false;
   }
 
   if (Symbol.isTemporary())
     return false;
 
   if (Symbol.getType() == ELF::STT_SECTION)
     return false;
 
   return true;
 }
 
 void ELFWriter::computeSymbolTable(
     MCAssembler &Asm, const MCAsmLayout &Layout,
     const SectionIndexMapTy &SectionIndexMap, const RevGroupMapTy &RevGroupMap,
     SectionOffsetsTy &SectionOffsets) {
   MCContext &Ctx = Asm.getContext();
   SymbolTableWriter Writer(*this, is64Bit());
 
   // Symbol table
   unsigned EntrySize = is64Bit() ? ELF::SYMENTRY_SIZE64 : ELF::SYMENTRY_SIZE32;
   MCSectionELF *SymtabSection =
       Ctx.getELFSection(".symtab", ELF::SHT_SYMTAB, 0, EntrySize);
   SymtabSection->setAlignment(is64Bit() ? Align(8) : Align(4));
   SymbolTableIndex = addToSectionTable(SymtabSection);
 
   uint64_t SecStart = align(SymtabSection->getAlignment());
 
   // The first entry is the undefined symbol entry.
   Writer.writeSymbol(0, 0, 0, 0, 0, 0, false);
 
   std::vector<ELFSymbolData> LocalSymbolData;
   std::vector<ELFSymbolData> ExternalSymbolData;
   MutableArrayRef<std::pair<std::string, size_t>> FileNames =
       Asm.getFileNames();
   for (const std::pair<std::string, size_t> &F : FileNames)
     StrTabBuilder.add(F.first);
 
   // Add the data for the symbols.
   bool HasLargeSectionIndex = false;
   for (auto It : llvm::enumerate(Asm.symbols())) {
     const auto &Symbol = cast<MCSymbolELF>(It.value());
     bool Used = Symbol.isUsedInReloc();
     bool WeakrefUsed = Symbol.isWeakrefUsedInReloc();
     bool isSignature = Symbol.isSignature();
 
     if (!isInSymtab(Layout, Symbol, Used || WeakrefUsed || isSignature,
                     OWriter.Renames.count(&Symbol)))
       continue;
 
     if (Symbol.isTemporary() && Symbol.isUndefined()) {
       Ctx.reportError(SMLoc(), "Undefined temporary symbol " + Symbol.getName());
       continue;
     }
 
     ELFSymbolData MSD;
     MSD.Symbol = cast<MCSymbolELF>(&Symbol);
     MSD.Order = It.index();
 
     bool Local = Symbol.getBinding() == ELF::STB_LOCAL;
     assert(Local || !Symbol.isTemporary());
 
     if (Symbol.isAbsolute()) {
       MSD.SectionIndex = ELF::SHN_ABS;
     } else if (Symbol.isCommon()) {
       if (Symbol.isTargetCommon()) {
         MSD.SectionIndex = Symbol.getIndex();
       } else {
         assert(!Local);
         MSD.SectionIndex = ELF::SHN_COMMON;
       }
     } else if (Symbol.isUndefined()) {
       if (isSignature && !Used) {
         MSD.SectionIndex = RevGroupMap.lookup(&Symbol);
         if (MSD.SectionIndex >= ELF::SHN_LORESERVE)
           HasLargeSectionIndex = true;
       } else {
         MSD.SectionIndex = ELF::SHN_UNDEF;
       }
     } else {
       const MCSectionELF &Section =
           static_cast<const MCSectionELF &>(Symbol.getSection());
 
       // We may end up with a situation when section symbol is technically
       // defined, but should not be. That happens because we explicitly
       // pre-create few .debug_* sections to have accessors.
       // And if these sections were not really defined in the code, but were
       // referenced, we simply error out.
       if (!Section.isRegistered()) {
         assert(static_cast<const MCSymbolELF &>(Symbol).getType() ==
                ELF::STT_SECTION);
         Ctx.reportError(SMLoc(),
                         "Undefined section reference: " + Symbol.getName());
         continue;
       }
 
       if (Mode == NonDwoOnly && isDwoSection(Section))
         continue;
       MSD.SectionIndex = SectionIndexMap.lookup(&Section);
       assert(MSD.SectionIndex && "Invalid section index!");
       if (MSD.SectionIndex >= ELF::SHN_LORESERVE)
         HasLargeSectionIndex = true;
     }
 
     StringRef Name = Symbol.getName();
 
     // Sections have their own string table
     if (Symbol.getType() != ELF::STT_SECTION) {
       MSD.Name = Name;
       StrTabBuilder.add(Name);
     }
 
     if (Local)
       LocalSymbolData.push_back(MSD);
     else
       ExternalSymbolData.push_back(MSD);
   }
 
   // This holds the .symtab_shndx section index.
   unsigned SymtabShndxSectionIndex = 0;
 
   if (HasLargeSectionIndex) {
     MCSectionELF *SymtabShndxSection =
         Ctx.getELFSection(".symtab_shndx", ELF::SHT_SYMTAB_SHNDX, 0, 4);
     SymtabShndxSectionIndex = addToSectionTable(SymtabShndxSection);
     SymtabShndxSection->setAlignment(Align(4));
   }
 
   StrTabBuilder.finalize();
 
   // Make the first STT_FILE precede previous local symbols.
   unsigned Index = 1;
   auto FileNameIt = FileNames.begin();
   if (!FileNames.empty())
     FileNames[0].second = 0;
 
   for (ELFSymbolData &MSD : LocalSymbolData) {
     // Emit STT_FILE symbols before their associated local symbols.
     for (; FileNameIt != FileNames.end() && FileNameIt->second <= MSD.Order;
          ++FileNameIt) {
       Writer.writeSymbol(StrTabBuilder.getOffset(FileNameIt->first),
                          ELF::STT_FILE | ELF::STB_LOCAL, 0, 0, ELF::STV_DEFAULT,
                          ELF::SHN_ABS, true);
       ++Index;
     }
 
     unsigned StringIndex = MSD.Symbol->getType() == ELF::STT_SECTION
                                ? 0
                                : StrTabBuilder.getOffset(MSD.Name);
     MSD.Symbol->setIndex(Index++);
     writeSymbol(Writer, StringIndex, MSD, Layout);
   }
   for (; FileNameIt != FileNames.end(); ++FileNameIt) {
     Writer.writeSymbol(StrTabBuilder.getOffset(FileNameIt->first),
                        ELF::STT_FILE | ELF::STB_LOCAL, 0, 0, ELF::STV_DEFAULT,
                        ELF::SHN_ABS, true);
     ++Index;
   }
 
   // Write the symbol table entries.
   LastLocalSymbolIndex = Index;
 
   for (ELFSymbolData &MSD : ExternalSymbolData) {
     unsigned StringIndex = StrTabBuilder.getOffset(MSD.Name);
     MSD.Symbol->setIndex(Index++);
     writeSymbol(Writer, StringIndex, MSD, Layout);
     assert(MSD.Symbol->getBinding() != ELF::STB_LOCAL);
   }
 
   uint64_t SecEnd = W.OS.tell();
   SectionOffsets[SymtabSection] = std::make_pair(SecStart, SecEnd);
 
   ArrayRef<uint32_t> ShndxIndexes = Writer.getShndxIndexes();
   if (ShndxIndexes.empty()) {
     assert(SymtabShndxSectionIndex == 0);
     return;
   }
   assert(SymtabShndxSectionIndex != 0);
 
   SecStart = W.OS.tell();
   const MCSectionELF *SymtabShndxSection =
       SectionTable[SymtabShndxSectionIndex - 1];
   for (uint32_t Index : ShndxIndexes)
     write(Index);
   SecEnd = W.OS.tell();
   SectionOffsets[SymtabShndxSection] = std::make_pair(SecStart, SecEnd);
 }
 
 void ELFWriter::writeAddrsigSection() {
   for (const MCSymbol *Sym : OWriter.AddrsigSyms)
     encodeULEB128(Sym->getIndex(), W.OS);
 }
 
 MCSectionELF *ELFWriter::createRelocationSection(MCContext &Ctx,
                                                  const MCSectionELF &Sec) {
   if (OWriter.Relocations[&Sec].empty())
     return nullptr;
 
   const StringRef SectionName = Sec.getName();
   bool Rela = usesRela(Sec);
   std::string RelaSectionName = Rela ? ".rela" : ".rel";
   RelaSectionName += SectionName;
 
   unsigned EntrySize;
   if (Rela)
     EntrySize = is64Bit() ? sizeof(ELF::Elf64_Rela) : sizeof(ELF::Elf32_Rela);
   else
     EntrySize = is64Bit() ? sizeof(ELF::Elf64_Rel) : sizeof(ELF::Elf32_Rel);
 
   unsigned Flags = ELF::SHF_INFO_LINK;
   if (Sec.getFlags() & ELF::SHF_GROUP)
     Flags = ELF::SHF_GROUP;
 
   MCSectionELF *RelaSection = Ctx.createELFRelSection(
       RelaSectionName, Rela ? ELF::SHT_RELA : ELF::SHT_REL, Flags, EntrySize,
       Sec.getGroup(), &Sec);
   RelaSection->setAlignment(is64Bit() ? Align(8) : Align(4));
   return RelaSection;
 }
 
 // Include the debug info compression header.
 bool ELFWriter::maybeWriteCompression(
     uint64_t Size, SmallVectorImpl<char> &CompressedContents, bool ZLibStyle,
     unsigned Alignment) {
   if (ZLibStyle) {
     uint64_t HdrSize =
         is64Bit() ? sizeof(ELF::Elf32_Chdr) : sizeof(ELF::Elf64_Chdr);
     if (Size <= HdrSize + CompressedContents.size())
       return false;
     // Platform specific header is followed by compressed data.
     if (is64Bit()) {
       // Write Elf64_Chdr header.
       write(static_cast<ELF::Elf64_Word>(ELF::ELFCOMPRESS_ZLIB));
       write(static_cast<ELF::Elf64_Word>(0)); // ch_reserved field.
       write(static_cast<ELF::Elf64_Xword>(Size));
       write(static_cast<ELF::Elf64_Xword>(Alignment));
     } else {
       // Write Elf32_Chdr header otherwise.
       write(static_cast<ELF::Elf32_Word>(ELF::ELFCOMPRESS_ZLIB));
       write(static_cast<ELF::Elf32_Word>(Size));
       write(static_cast<ELF::Elf32_Word>(Alignment));
     }
     return true;
   }
 
   // "ZLIB" followed by 8 bytes representing the uncompressed size of the section,
   // useful for consumers to preallocate a buffer to decompress into.
   const StringRef Magic = "ZLIB";
   if (Size <= Magic.size() + sizeof(Size) + CompressedContents.size())
     return false;
   W.OS << Magic;
   support::endian::write(W.OS, Size, support::big);
   return true;
 }
 
 void ELFWriter::writeSectionData(const MCAssembler &Asm, MCSection &Sec,
                                  const MCAsmLayout &Layout) {
   MCSectionELF &Section = static_cast<MCSectionELF &>(Sec);
   StringRef SectionName = Section.getName();
 
   auto &MC = Asm.getContext();
   const auto &MAI = MC.getAsmInfo();
 
   // Compressing debug_frame requires handling alignment fragments which is
   // more work (possibly generalizing MCAssembler.cpp:writeFragment to allow
   // for writing to arbitrary buffers) for little benefit.
   bool CompressionEnabled =
       MAI->compressDebugSections() != DebugCompressionType::None;
   if (!CompressionEnabled || !SectionName.startswith(".debug_") ||
       SectionName == ".debug_frame") {
     Asm.writeSectionData(W.OS, &Section, Layout);
     return;
   }
 
   assert((MAI->compressDebugSections() == DebugCompressionType::Z ||
           MAI->compressDebugSections() == DebugCompressionType::GNU) &&
          "expected zlib or zlib-gnu style compression");
 
   SmallVector<char, 128> UncompressedData;
   raw_svector_ostream VecOS(UncompressedData);
   Asm.writeSectionData(VecOS, &Section, Layout);
 
   SmallVector<char, 128> CompressedContents;
   if (Error E = zlib::compress(
           StringRef(UncompressedData.data(), UncompressedData.size()),
           CompressedContents)) {
     consumeError(std::move(E));
     W.OS << UncompressedData;
     return;
   }
 
   bool ZlibStyle = MAI->compressDebugSections() == DebugCompressionType::Z;
   if (!maybeWriteCompression(UncompressedData.size(), CompressedContents,
                              ZlibStyle, Sec.getAlignment())) {
     W.OS << UncompressedData;
     return;
   }
 
   if (ZlibStyle) {
     // Set the compressed flag. That is zlib style.
     Section.setFlags(Section.getFlags() | ELF::SHF_COMPRESSED);
     // Alignment field should reflect the requirements of
     // the compressed section header.
     Section.setAlignment(is64Bit() ? Align(8) : Align(4));
   } else {
     // Add "z" prefix to section name. This is zlib-gnu style.
     MC.renameELFSection(&Section, (".z" + SectionName.drop_front(1)).str());
   }
   W.OS << CompressedContents;
 }
 
 void ELFWriter::WriteSecHdrEntry(uint32_t Name, uint32_t Type, uint64_t Flags,
                                  uint64_t Address, uint64_t Offset,
                                  uint64_t Size, uint32_t Link, uint32_t Info,
                                  uint64_t Alignment, uint64_t EntrySize) {
   W.write<uint32_t>(Name);        // sh_name: index into string table
   W.write<uint32_t>(Type);        // sh_type
   WriteWord(Flags);     // sh_flags
   WriteWord(Address);   // sh_addr
   WriteWord(Offset);    // sh_offset
   WriteWord(Size);      // sh_size
   W.write<uint32_t>(Link);        // sh_link
   W.write<uint32_t>(Info);        // sh_info
   WriteWord(Alignment); // sh_addralign
   WriteWord(EntrySize); // sh_entsize
 }
 
 void ELFWriter::writeRelocations(const MCAssembler &Asm,
                                        const MCSectionELF &Sec) {
   std::vector<ELFRelocationEntry> &Relocs = OWriter.Relocations[&Sec];
 
   // We record relocations by pushing to the end of a vector. Reverse the vector
   // to get the relocations in the order they were created.
   // In most cases that is not important, but it can be for special sections
   // (.eh_frame) or specific relocations (TLS optimizations on SystemZ).
   std::reverse(Relocs.begin(), Relocs.end());
 
   // Sort the relocation entries. MIPS needs this.
   OWriter.TargetObjectWriter->sortRelocs(Asm, Relocs);
 
   const bool Rela = usesRela(Sec);
   for (unsigned i = 0, e = Relocs.size(); i != e; ++i) {
     const ELFRelocationEntry &Entry = Relocs[e - i - 1];
     unsigned Index = Entry.Symbol ? Entry.Symbol->getIndex() : 0;
 
     if (is64Bit()) {
       write(Entry.Offset);
       if (OWriter.TargetObjectWriter->getEMachine() == ELF::EM_MIPS) {
         write(uint32_t(Index));
 
         write(OWriter.TargetObjectWriter->getRSsym(Entry.Type));
         write(OWriter.TargetObjectWriter->getRType3(Entry.Type));
         write(OWriter.TargetObjectWriter->getRType2(Entry.Type));
         write(OWriter.TargetObjectWriter->getRType(Entry.Type));
       } else {
         struct ELF::Elf64_Rela ERE64;
         ERE64.setSymbolAndType(Index, Entry.Type);
         write(ERE64.r_info);
       }
       if (Rela)
         write(Entry.Addend);
     } else {
       write(uint32_t(Entry.Offset));
 
       struct ELF::Elf32_Rela ERE32;
       ERE32.setSymbolAndType(Index, Entry.Type);
       write(ERE32.r_info);
 
       if (Rela)
         write(uint32_t(Entry.Addend));
 
       if (OWriter.TargetObjectWriter->getEMachine() == ELF::EM_MIPS) {
         if (uint32_t RType =
                 OWriter.TargetObjectWriter->getRType2(Entry.Type)) {
           write(uint32_t(Entry.Offset));
 
           ERE32.setSymbolAndType(0, RType);
           write(ERE32.r_info);
           write(uint32_t(0));
         }
         if (uint32_t RType =
                 OWriter.TargetObjectWriter->getRType3(Entry.Type)) {
           write(uint32_t(Entry.Offset));
 
           ERE32.setSymbolAndType(0, RType);
           write(ERE32.r_info);
           write(uint32_t(0));
         }
       }
     }
   }
 }
 
 void ELFWriter::writeSection(const SectionIndexMapTy &SectionIndexMap,
                              uint32_t GroupSymbolIndex, uint64_t Offset,
                              uint64_t Size, const MCSectionELF &Section) {
   uint64_t sh_link = 0;
   uint64_t sh_info = 0;
 
   switch(Section.getType()) {
   default:
     // Nothing to do.
     break;
 
   case ELF::SHT_DYNAMIC:
     llvm_unreachable("SHT_DYNAMIC in a relocatable object");
 
   case ELF::SHT_REL:
   case ELF::SHT_RELA: {
     sh_link = SymbolTableIndex;
     assert(sh_link && ".symtab not found");
     const MCSection *InfoSection = Section.getLinkedToSection();
     sh_info = SectionIndexMap.lookup(cast<MCSectionELF>(InfoSection));
     break;
   }
 
   case ELF::SHT_SYMTAB:
     sh_link = StringTableIndex;
     sh_info = LastLocalSymbolIndex;
     break;
 
   case ELF::SHT_SYMTAB_SHNDX:
   case ELF::SHT_LLVM_CALL_GRAPH_PROFILE:
   case ELF::SHT_LLVM_ADDRSIG:
     sh_link = SymbolTableIndex;
     break;
 
   case ELF::SHT_GROUP:
     sh_link = SymbolTableIndex;
     sh_info = GroupSymbolIndex;
     break;
   }
 
   if (Section.getFlags() & ELF::SHF_LINK_ORDER) {
     // If the value in the associated metadata is not a definition, Sym will be
     // undefined. Represent this with sh_link=0.
     const MCSymbol *Sym = Section.getLinkedToSymbol();
     if (Sym && Sym->isInSection()) {
       const MCSectionELF *Sec = cast<MCSectionELF>(&Sym->getSection());
       sh_link = SectionIndexMap.lookup(Sec);
     }
   }
 
   WriteSecHdrEntry(StrTabBuilder.getOffset(Section.getName()),
                    Section.getType(), Section.getFlags(), 0, Offset, Size,
                    sh_link, sh_info, Section.getAlignment(),
                    Section.getEntrySize());
 }
 
 void ELFWriter::writeSectionHeader(
     const MCAsmLayout &Layout, const SectionIndexMapTy &SectionIndexMap,
     const SectionOffsetsTy &SectionOffsets) {
   const unsigned NumSections = SectionTable.size();
 
   // Null section first.
   uint64_t FirstSectionSize =
       (NumSections + 1) >= ELF::SHN_LORESERVE ? NumSections + 1 : 0;
   WriteSecHdrEntry(0, 0, 0, 0, 0, FirstSectionSize, 0, 0, 0, 0);
 
   for (const MCSectionELF *Section : SectionTable) {
     uint32_t GroupSymbolIndex;
     unsigned Type = Section->getType();
     if (Type != ELF::SHT_GROUP)
       GroupSymbolIndex = 0;
     else
       GroupSymbolIndex = Section->getGroup()->getIndex();
 
     const std::pair<uint64_t, uint64_t> &Offsets =
         SectionOffsets.find(Section)->second;
     uint64_t Size;
     if (Type == ELF::SHT_NOBITS)
       Size = Layout.getSectionAddressSize(Section);
     else
       Size = Offsets.second - Offsets.first;
 
     writeSection(SectionIndexMap, GroupSymbolIndex, Offsets.first, Size,
                  *Section);
   }
 }
 
 uint64_t ELFWriter::writeObject(MCAssembler &Asm, const MCAsmLayout &Layout) {
   uint64_t StartOffset = W.OS.tell();
 
   MCContext &Ctx = Asm.getContext();
   MCSectionELF *StrtabSection =
       Ctx.getELFSection(".strtab", ELF::SHT_STRTAB, 0);
   StringTableIndex = addToSectionTable(StrtabSection);
 
   RevGroupMapTy RevGroupMap;
   SectionIndexMapTy SectionIndexMap;
 
   std::map<const MCSymbol *, std::vector<const MCSectionELF *>> GroupMembers;
 
   // Write out the ELF header ...
   writeHeader(Asm);
 
   // ... then the sections ...
   SectionOffsetsTy SectionOffsets;
   std::vector<MCSectionELF *> Groups;
   std::vector<MCSectionELF *> Relocations;
   for (MCSection &Sec : Asm) {
     MCSectionELF &Section = static_cast<MCSectionELF &>(Sec);
     if (Mode == NonDwoOnly && isDwoSection(Section))
       continue;
     if (Mode == DwoOnly && !isDwoSection(Section))
       continue;
 
     // Remember the offset into the file for this section.
     const uint64_t SecStart = align(Section.getAlignment());
 
     const MCSymbolELF *SignatureSymbol = Section.getGroup();
     writeSectionData(Asm, Section, Layout);
 
     uint64_t SecEnd = W.OS.tell();
     SectionOffsets[&Section] = std::make_pair(SecStart, SecEnd);
 
     MCSectionELF *RelSection = createRelocationSection(Ctx, Section);
 
     if (SignatureSymbol) {
       unsigned &GroupIdx = RevGroupMap[SignatureSymbol];
       if (!GroupIdx) {
         MCSectionELF *Group =
             Ctx.createELFGroupSection(SignatureSymbol, Section.isComdat());
         GroupIdx = addToSectionTable(Group);
         Group->setAlignment(Align(4));
         Groups.push_back(Group);
       }
       std::vector<const MCSectionELF *> &Members =
           GroupMembers[SignatureSymbol];
       Members.push_back(&Section);
       if (RelSection)
         Members.push_back(RelSection);
     }
 
     SectionIndexMap[&Section] = addToSectionTable(&Section);
     if (RelSection) {
       SectionIndexMap[RelSection] = addToSectionTable(RelSection);
       Relocations.push_back(RelSection);
     }
 
     OWriter.TargetObjectWriter->addTargetSectionFlags(Ctx, Section);
   }
 
   for (MCSectionELF *Group : Groups) {
     // Remember the offset into the file for this section.
     const uint64_t SecStart = align(Group->getAlignment());
 
     const MCSymbol *SignatureSymbol = Group->getGroup();
     assert(SignatureSymbol);
     write(uint32_t(Group->isComdat() ? unsigned(ELF::GRP_COMDAT) : 0));
     for (const MCSectionELF *Member : GroupMembers[SignatureSymbol]) {
       uint32_t SecIndex = SectionIndexMap.lookup(Member);
       write(SecIndex);
     }
 
     uint64_t SecEnd = W.OS.tell();
     SectionOffsets[Group] = std::make_pair(SecStart, SecEnd);
   }
 
   if (Mode == DwoOnly) {
     // dwo files don't have symbol tables or relocations, but they do have
     // string tables.
     StrTabBuilder.finalize();
   } else {
     MCSectionELF *AddrsigSection;
     if (OWriter.EmitAddrsigSection) {
       AddrsigSection = Ctx.getELFSection(".llvm_addrsig", ELF::SHT_LLVM_ADDRSIG,
                                          ELF::SHF_EXCLUDE);
       addToSectionTable(AddrsigSection);
     }
 
     // Compute symbol table information.
     computeSymbolTable(Asm, Layout, SectionIndexMap, RevGroupMap,
                        SectionOffsets);
 
     for (MCSectionELF *RelSection : Relocations) {
       // Remember the offset into the file for this section.
       const uint64_t SecStart = align(RelSection->getAlignment());
 
       writeRelocations(Asm,
                        cast<MCSectionELF>(*RelSection->getLinkedToSection()));
 
       uint64_t SecEnd = W.OS.tell();
       SectionOffsets[RelSection] = std::make_pair(SecStart, SecEnd);
     }
 
     if (OWriter.EmitAddrsigSection) {
       uint64_t SecStart = W.OS.tell();
       writeAddrsigSection();
       uint64_t SecEnd = W.OS.tell();
       SectionOffsets[AddrsigSection] = std::make_pair(SecStart, SecEnd);
     }
   }
 
   {
     uint64_t SecStart = W.OS.tell();
     StrTabBuilder.write(W.OS);
     SectionOffsets[StrtabSection] = std::make_pair(SecStart, W.OS.tell());
   }
 
   const uint64_t SectionHeaderOffset = align(is64Bit() ? 8 : 4);
 
   // ... then the section header table ...
   writeSectionHeader(Layout, SectionIndexMap, SectionOffsets);
 
   uint16_t NumSections = support::endian::byte_swap<uint16_t>(
       (SectionTable.size() + 1 >= ELF::SHN_LORESERVE) ? (uint16_t)ELF::SHN_UNDEF
                                                       : SectionTable.size() + 1,
       W.Endian);
   unsigned NumSectionsOffset;
 
   auto &Stream = static_cast<raw_pwrite_stream &>(W.OS);
   if (is64Bit()) {
     uint64_t Val =
         support::endian::byte_swap<uint64_t>(SectionHeaderOffset, W.Endian);
     Stream.pwrite(reinterpret_cast<char *>(&Val), sizeof(Val),
                   offsetof(ELF::Elf64_Ehdr, e_shoff));
     NumSectionsOffset = offsetof(ELF::Elf64_Ehdr, e_shnum);
   } else {
     uint32_t Val =
         support::endian::byte_swap<uint32_t>(SectionHeaderOffset, W.Endian);
     Stream.pwrite(reinterpret_cast<char *>(&Val), sizeof(Val),
                   offsetof(ELF::Elf32_Ehdr, e_shoff));
     NumSectionsOffset = offsetof(ELF::Elf32_Ehdr, e_shnum);
   }
   Stream.pwrite(reinterpret_cast<char *>(&NumSections), sizeof(NumSections),
                 NumSectionsOffset);
 
   return W.OS.tell() - StartOffset;
 }
 
 bool ELFObjectWriter::hasRelocationAddend() const {
   return TargetObjectWriter->hasRelocationAddend();
 }
 
 void ELFObjectWriter::executePostLayoutBinding(MCAssembler &Asm,
                                                const MCAsmLayout &Layout) {
   // The presence of symbol versions causes undefined symbols and
   // versions declared with @@@ to be renamed.
   for (const MCAssembler::Symver &S : Asm.Symvers) {
     StringRef AliasName = S.Name;
     const auto &Symbol = cast<MCSymbolELF>(*S.Sym);
     size_t Pos = AliasName.find('@');
     assert(Pos != StringRef::npos);
 
     StringRef Prefix = AliasName.substr(0, Pos);
     StringRef Rest = AliasName.substr(Pos);
     StringRef Tail = Rest;
     if (Rest.startswith("@@@"))
       Tail = Rest.substr(Symbol.isUndefined() ? 2 : 1);
 
     auto *Alias =
         cast<MCSymbolELF>(Asm.getContext().getOrCreateSymbol(Prefix + Tail));
     Asm.registerSymbol(*Alias);
     const MCExpr *Value = MCSymbolRefExpr::create(&Symbol, Asm.getContext());
     Alias->setVariableValue(Value);
 
     // Aliases defined with .symvar copy the binding from the symbol they alias.
     // This is the first place we are able to copy this information.
     Alias->setBinding(Symbol.getBinding());
     Alias->setVisibility(Symbol.getVisibility());
     Alias->setOther(Symbol.getOther());
 
     if (!Symbol.isUndefined() && S.KeepOriginalSym)
       continue;
 
     if (Symbol.isUndefined() && Rest.startswith("@@") &&
         !Rest.startswith("@@@")) {
       Asm.getContext().reportError(S.Loc, "default version symbol " +
                                               AliasName + " must be defined");
       continue;
     }
 
     if (Renames.count(&Symbol) && Renames[&Symbol] != Alias) {
       Asm.getContext().reportError(S.Loc, Twine("multiple versions for ") +
                                               Symbol.getName());
       continue;
     }
 
     Renames.insert(std::make_pair(&Symbol, Alias));
   }
 
   for (const MCSymbol *&Sym : AddrsigSyms) {
     if (const MCSymbol *R = Renames.lookup(cast<MCSymbolELF>(Sym)))
       Sym = R;
     if (Sym->isInSection() && Sym->getName().startswith(".L"))
       Sym = Sym->getSection().getBeginSymbol();
     Sym->setUsedInReloc();
   }
 }
 
 // It is always valid to create a relocation with a symbol. It is preferable
 // to use a relocation with a section if that is possible. Using the section
 // allows us to omit some local symbols from the symbol table.
 bool ELFObjectWriter::shouldRelocateWithSymbol(const MCAssembler &Asm,
                                                const MCSymbolRefExpr *RefA,
                                                const MCSymbolELF *Sym,
                                                uint64_t C,
                                                unsigned Type) const {
   // A PCRel relocation to an absolute value has no symbol (or section). We
   // represent that with a relocation to a null section.
   if (!RefA)
     return false;
 
   MCSymbolRefExpr::VariantKind Kind = RefA->getKind();
   switch (Kind) {
   default:
     break;
   // The .odp creation emits a relocation against the symbol ".TOC." which
   // create a R_PPC64_TOC relocation. However the relocation symbol name
   // in final object creation should be NULL, since the symbol does not
   // really exist, it is just the reference to TOC base for the current
   // object file. Since the symbol is undefined, returning false results
   // in a relocation with a null section which is the desired result.
   case MCSymbolRefExpr::VK_PPC_TOCBASE:
     return false;
 
   // These VariantKind cause the relocation to refer to something other than
   // the symbol itself, like a linker generated table. Since the address of
   // symbol is not relevant, we cannot replace the symbol with the
   // section and patch the difference in the addend.
   case MCSymbolRefExpr::VK_GOT:
   case MCSymbolRefExpr::VK_PLT:
   case MCSymbolRefExpr::VK_GOTPCREL:
   case MCSymbolRefExpr::VK_GOTPCREL_NORELAX:
   case MCSymbolRefExpr::VK_PPC_GOT_LO:
   case MCSymbolRefExpr::VK_PPC_GOT_HI:
   case MCSymbolRefExpr::VK_PPC_GOT_HA:
     return true;
   }
 
   // An undefined symbol is not in any section, so the relocation has to point
   // to the symbol itself.
   assert(Sym && "Expected a symbol");
   if (Sym->isUndefined())
     return true;
 
   unsigned Binding = Sym->getBinding();
   switch(Binding) {
   default:
     llvm_unreachable("Invalid Binding");
   case ELF::STB_LOCAL:
     break;
   case ELF::STB_WEAK:
     // If the symbol is weak, it might be overridden by a symbol in another
     // file. The relocation has to point to the symbol so that the linker
     // can update it.
     return true;
   case ELF::STB_GLOBAL:
   case ELF::STB_GNU_UNIQUE:
     // Global ELF symbols can be preempted by the dynamic linker. The relocation
     // has to point to the symbol for a reason analogous to the STB_WEAK case.
     return true;
   }
 
   // Keep symbol type for a local ifunc because it may result in an IRELATIVE
   // reloc that the dynamic loader will use to resolve the address at startup
   // time.
   if (Sym->getType() == ELF::STT_GNU_IFUNC)
     return true;
 
   // If a relocation points to a mergeable section, we have to be careful.
   // If the offset is zero, a relocation with the section will encode the
   // same information. With a non-zero offset, the situation is different.
   // For example, a relocation can point 42 bytes past the end of a string.
   // If we change such a relocation to use the section, the linker would think
   // that it pointed to another string and subtracting 42 at runtime will
   // produce the wrong value.
   if (Sym->isInSection()) {
     auto &Sec = cast<MCSectionELF>(Sym->getSection());
     unsigned Flags = Sec.getFlags();
     if (Flags & ELF::SHF_MERGE) {
       if (C != 0)
         return true;
 
       // gold<2.34 incorrectly ignored the addend for R_386_GOTOFF (9)
       // (http://sourceware.org/PR16794).
       if (TargetObjectWriter->getEMachine() == ELF::EM_386 &&
           Type == ELF::R_386_GOTOFF)
         return true;
 
       // ld.lld handles R_MIPS_HI16/R_MIPS_LO16 separately, not as a whole, so
       // it doesn't know that an R_MIPS_HI16 with implicit addend 1 and an
       // R_MIPS_LO16 with implicit addend -32768 represents 32768, which is in
       // range of a MergeInputSection. We could introduce a new RelExpr member
       // (like R_RISCV_PC_INDIRECT for R_RISCV_PCREL_HI20 / R_RISCV_PCREL_LO12)
       // but the complexity is unnecessary given that GNU as keeps the original
       // symbol for this case as well.
       if (TargetObjectWriter->getEMachine() == ELF::EM_MIPS &&
           !hasRelocationAddend())
         return true;
     }
 
     // Most TLS relocations use a got, so they need the symbol. Even those that
     // are just an offset (@tpoff), require a symbol in gold versions before
     // 5efeedf61e4fe720fd3e9a08e6c91c10abb66d42 (2014-09-26) which fixed
     // http://sourceware.org/PR16773.
     if (Flags & ELF::SHF_TLS)
       return true;
   }
 
   // If the symbol is a thumb function the final relocation must set the lowest
   // bit. With a symbol that is done by just having the symbol have that bit
   // set, so we would lose the bit if we relocated with the section.
   // FIXME: We could use the section but add the bit to the relocation value.
   if (Asm.isThumbFunc(Sym))
     return true;
 
   if (TargetObjectWriter->needsRelocateWithSymbol(*Sym, Type))
     return true;
   return false;
 }
 
 void ELFObjectWriter::recordRelocation(MCAssembler &Asm,
                                        const MCAsmLayout &Layout,
                                        const MCFragment *Fragment,
                                        const MCFixup &Fixup, MCValue Target,
                                        uint64_t &FixedValue) {
   MCAsmBackend &Backend = Asm.getBackend();
   bool IsPCRel = Backend.getFixupKindInfo(Fixup.getKind()).Flags &
                  MCFixupKindInfo::FKF_IsPCRel;
   const MCSectionELF &FixupSection = cast<MCSectionELF>(*Fragment->getParent());
   uint64_t C = Target.getConstant();
   uint64_t FixupOffset = Layout.getFragmentOffset(Fragment) + Fixup.getOffset();
   MCContext &Ctx = Asm.getContext();
 
   if (const MCSymbolRefExpr *RefB = Target.getSymB()) {
     const auto &SymB = cast<MCSymbolELF>(RefB->getSymbol());
     if (SymB.isUndefined()) {
       Ctx.reportError(Fixup.getLoc(),
                       Twine("symbol '") + SymB.getName() +
                           "' can not be undefined in a subtraction expression");
       return;
     }
 
     assert(!SymB.isAbsolute() && "Should have been folded");
     const MCSection &SecB = SymB.getSection();
     if (&SecB != &FixupSection) {
       Ctx.reportError(Fixup.getLoc(),
                       "Cannot represent a difference across sections");
       return;
     }
 
     assert(!IsPCRel && "should have been folded");
     IsPCRel = true;
     C += FixupOffset - Layout.getSymbolOffset(SymB);
   }
 
   // We either rejected the fixup or folded B into C at this point.
   const MCSymbolRefExpr *RefA = Target.getSymA();
   const auto *SymA = RefA ? cast<MCSymbolELF>(&RefA->getSymbol()) : nullptr;
 
   bool ViaWeakRef = false;
   if (SymA && SymA->isVariable()) {
     const MCExpr *Expr = SymA->getVariableValue();
     if (const auto *Inner = dyn_cast<MCSymbolRefExpr>(Expr)) {
       if (Inner->getKind() == MCSymbolRefExpr::VK_WEAKREF) {
         SymA = cast<MCSymbolELF>(&Inner->getSymbol());
         ViaWeakRef = true;
       }
     }
   }
 
   const MCSectionELF *SecA = (SymA && SymA->isInSection())
                                  ? cast<MCSectionELF>(&SymA->getSection())
                                  : nullptr;
   if (!checkRelocation(Ctx, Fixup.getLoc(), &FixupSection, SecA))
     return;
 
   unsigned Type = TargetObjectWriter->getRelocType(Ctx, Target, Fixup, IsPCRel);
   const auto *Parent = cast<MCSectionELF>(Fragment->getParent());
   // Emiting relocation with sybmol for CG Profile to  help with --cg-profile.
   bool RelocateWithSymbol =
       shouldRelocateWithSymbol(Asm, RefA, SymA, C, Type) ||
       (Parent->getType() == ELF::SHT_LLVM_CALL_GRAPH_PROFILE);
   uint64_t Addend = 0;
 
   FixedValue = !RelocateWithSymbol && SymA && !SymA->isUndefined()
                    ? C + Layout.getSymbolOffset(*SymA)
                    : C;
   if (hasRelocationAddend()) {
     Addend = FixedValue;
     FixedValue = 0;
   }
 
   if (!RelocateWithSymbol) {
     const auto *SectionSymbol =
         SecA ? cast<MCSymbolELF>(SecA->getBeginSymbol()) : nullptr;
     if (SectionSymbol)
       SectionSymbol->setUsedInReloc();
     ELFRelocationEntry Rec(FixupOffset, SectionSymbol, Type, Addend, SymA, C);
     Relocations[&FixupSection].push_back(Rec);
     return;
   }
 
   const MCSymbolELF *RenamedSymA = SymA;
   if (SymA) {
     if (const MCSymbolELF *R = Renames.lookup(SymA))
       RenamedSymA = R;
 
     if (ViaWeakRef)
       RenamedSymA->setIsWeakrefUsedInReloc();
     else
       RenamedSymA->setUsedInReloc();
   }
   ELFRelocationEntry Rec(FixupOffset, RenamedSymA, Type, Addend, SymA, C);
   Relocations[&FixupSection].push_back(Rec);
 }
 
 bool ELFObjectWriter::isSymbolRefDifferenceFullyResolvedImpl(
     const MCAssembler &Asm, const MCSymbol &SA, const MCFragment &FB,
     bool InSet, bool IsPCRel) const {
   const auto &SymA = cast<MCSymbolELF>(SA);
   if (IsPCRel) {
     assert(!InSet);
     if (SymA.getBinding() != ELF::STB_LOCAL ||
         SymA.getType() == ELF::STT_GNU_IFUNC)
       return false;
   }
   return MCObjectWriter::isSymbolRefDifferenceFullyResolvedImpl(Asm, SymA, FB,
                                                                 InSet, IsPCRel);
 }
 
 std::unique_ptr<MCObjectWriter>
 llvm::createELFObjectWriter(std::unique_ptr<MCELFObjectTargetWriter> MOTW,
                             raw_pwrite_stream &OS, bool IsLittleEndian) {
   return std::make_unique<ELFSingleObjectWriter>(std::move(MOTW), OS,
                                                   IsLittleEndian);
 }
 
 std::unique_ptr<MCObjectWriter>
 llvm::createELFDwoObjectWriter(std::unique_ptr<MCELFObjectTargetWriter> MOTW,
                                raw_pwrite_stream &OS, raw_pwrite_stream &DwoOS,
                                bool IsLittleEndian) {
   return std::make_unique<ELFDwoObjectWriter>(std::move(MOTW), OS, DwoOS,
                                                IsLittleEndian);
 }
diff --git a/contrib/llvm-project/llvm/lib/Support/Host.cpp b/contrib/llvm-project/llvm/lib/Support/Host.cpp
index f6003b783245..a82a4d451c8a 100644
--- a/contrib/llvm-project/llvm/lib/Support/Host.cpp
+++ b/contrib/llvm-project/llvm/lib/Support/Host.cpp
@@ -1,1770 +1,1776 @@
 //===-- Host.cpp - Implement OS Host Concept --------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //  This file implements the operating system Host concept.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/Host.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/BCD.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/X86TargetParser.h"
 #include "llvm/Support/raw_ostream.h"
 #include <assert.h>
 #include <string.h>
 
 // Include the platform-specific parts of this class.
 #ifdef LLVM_ON_UNIX
 #include "Unix/Host.inc"
 #include <sched.h>
 #endif
 #ifdef _WIN32
 #include "Windows/Host.inc"
 #endif
 #ifdef _MSC_VER
 #include <intrin.h>
 #endif
 #if defined(__APPLE__) && (!defined(__x86_64__))
 #include <mach/host_info.h>
 #include <mach/mach.h>
 #include <mach/mach_host.h>
 #include <mach/machine.h>
 #endif
 #ifdef _AIX
 #include <sys/systemcfg.h>
 #endif
 
 #define DEBUG_TYPE "host-detection"
 
 //===----------------------------------------------------------------------===//
 //
 //  Implementations of the CPU detection routines
 //
 //===----------------------------------------------------------------------===//
 
 using namespace llvm;
 
 static std::unique_ptr<llvm::MemoryBuffer>
     LLVM_ATTRIBUTE_UNUSED getProcCpuinfoContent() {
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Text =
       llvm::MemoryBuffer::getFileAsStream("/proc/cpuinfo");
   if (std::error_code EC = Text.getError()) {
     llvm::errs() << "Can't read "
                  << "/proc/cpuinfo: " << EC.message() << "\n";
     return nullptr;
   }
   return std::move(*Text);
 }
 
 StringRef sys::detail::getHostCPUNameForPowerPC(StringRef ProcCpuinfoContent) {
   // Access to the Processor Version Register (PVR) on PowerPC is privileged,
   // and so we must use an operating-system interface to determine the current
   // processor type. On Linux, this is exposed through the /proc/cpuinfo file.
   const char *generic = "generic";
 
   // The cpu line is second (after the 'processor: 0' line), so if this
   // buffer is too small then something has changed (or is wrong).
   StringRef::const_iterator CPUInfoStart = ProcCpuinfoContent.begin();
   StringRef::const_iterator CPUInfoEnd = ProcCpuinfoContent.end();
 
   StringRef::const_iterator CIP = CPUInfoStart;
 
   StringRef::const_iterator CPUStart = nullptr;
   size_t CPULen = 0;
 
   // We need to find the first line which starts with cpu, spaces, and a colon.
   // After the colon, there may be some additional spaces and then the cpu type.
   while (CIP < CPUInfoEnd && CPUStart == nullptr) {
     if (CIP < CPUInfoEnd && *CIP == '\n')
       ++CIP;
 
     if (CIP < CPUInfoEnd && *CIP == 'c') {
       ++CIP;
       if (CIP < CPUInfoEnd && *CIP == 'p') {
         ++CIP;
         if (CIP < CPUInfoEnd && *CIP == 'u') {
           ++CIP;
           while (CIP < CPUInfoEnd && (*CIP == ' ' || *CIP == '\t'))
             ++CIP;
 
           if (CIP < CPUInfoEnd && *CIP == ':') {
             ++CIP;
             while (CIP < CPUInfoEnd && (*CIP == ' ' || *CIP == '\t'))
               ++CIP;
 
             if (CIP < CPUInfoEnd) {
               CPUStart = CIP;
               while (CIP < CPUInfoEnd && (*CIP != ' ' && *CIP != '\t' &&
                                           *CIP != ',' && *CIP != '\n'))
                 ++CIP;
               CPULen = CIP - CPUStart;
             }
           }
         }
       }
     }
 
     if (CPUStart == nullptr)
       while (CIP < CPUInfoEnd && *CIP != '\n')
         ++CIP;
   }
 
   if (CPUStart == nullptr)
     return generic;
 
   return StringSwitch<const char *>(StringRef(CPUStart, CPULen))
       .Case("604e", "604e")
       .Case("604", "604")
       .Case("7400", "7400")
       .Case("7410", "7400")
       .Case("7447", "7400")
       .Case("7455", "7450")
       .Case("G4", "g4")
       .Case("POWER4", "970")
       .Case("PPC970FX", "970")
       .Case("PPC970MP", "970")
       .Case("G5", "g5")
       .Case("POWER5", "g5")
       .Case("A2", "a2")
       .Case("POWER6", "pwr6")
       .Case("POWER7", "pwr7")
       .Case("POWER8", "pwr8")
       .Case("POWER8E", "pwr8")
       .Case("POWER8NVL", "pwr8")
       .Case("POWER9", "pwr9")
       .Case("POWER10", "pwr10")
       // FIXME: If we get a simulator or machine with the capabilities of
       // mcpu=future, we should revisit this and add the name reported by the
       // simulator/machine.
       .Default(generic);
 }
 
 StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) {
   // The cpuid register on arm is not accessible from user space. On Linux,
   // it is exposed through the /proc/cpuinfo file.
 
   // Read 32 lines from /proc/cpuinfo, which should contain the CPU part line
   // in all cases.
   SmallVector<StringRef, 32> Lines;
   ProcCpuinfoContent.split(Lines, "\n");
 
   // Look for the CPU implementer line.
   StringRef Implementer;
   StringRef Hardware;
   StringRef Part;
   for (unsigned I = 0, E = Lines.size(); I != E; ++I) {
     if (Lines[I].startswith("CPU implementer"))
       Implementer = Lines[I].substr(15).ltrim("\t :");
     if (Lines[I].startswith("Hardware"))
       Hardware = Lines[I].substr(8).ltrim("\t :");
     if (Lines[I].startswith("CPU part"))
       Part = Lines[I].substr(8).ltrim("\t :");
   }
 
   if (Implementer == "0x41") { // ARM Ltd.
     // MSM8992/8994 may give cpu part for the core that the kernel is running on,
     // which is undeterministic and wrong. Always return cortex-a53 for these SoC.
     if (Hardware.endswith("MSM8994") || Hardware.endswith("MSM8996"))
       return "cortex-a53";
 
 
     // The CPU part is a 3 digit hexadecimal number with a 0x prefix. The
     // values correspond to the "Part number" in the CP15/c0 register. The
     // contents are specified in the various processor manuals.
     // This corresponds to the Main ID Register in Technical Reference Manuals.
     // and is used in programs like sys-utils
     return StringSwitch<const char *>(Part)
         .Case("0x926", "arm926ej-s")
         .Case("0xb02", "mpcore")
         .Case("0xb36", "arm1136j-s")
         .Case("0xb56", "arm1156t2-s")
         .Case("0xb76", "arm1176jz-s")
         .Case("0xc08", "cortex-a8")
         .Case("0xc09", "cortex-a9")
         .Case("0xc0f", "cortex-a15")
         .Case("0xc20", "cortex-m0")
         .Case("0xc23", "cortex-m3")
         .Case("0xc24", "cortex-m4")
         .Case("0xd22", "cortex-m55")
         .Case("0xd02", "cortex-a34")
         .Case("0xd04", "cortex-a35")
         .Case("0xd03", "cortex-a53")
         .Case("0xd07", "cortex-a57")
         .Case("0xd08", "cortex-a72")
         .Case("0xd09", "cortex-a73")
         .Case("0xd0a", "cortex-a75")
         .Case("0xd0b", "cortex-a76")
         .Case("0xd0d", "cortex-a77")
         .Case("0xd41", "cortex-a78")
         .Case("0xd44", "cortex-x1")
         .Case("0xd4c", "cortex-x1c")
         .Case("0xd0c", "neoverse-n1")
         .Case("0xd49", "neoverse-n2")
         .Case("0xd40", "neoverse-v1")
         .Default("generic");
   }
 
   if (Implementer == "0x42" || Implementer == "0x43") { // Broadcom | Cavium.
     return StringSwitch<const char *>(Part)
       .Case("0x516", "thunderx2t99")
       .Case("0x0516", "thunderx2t99")
       .Case("0xaf", "thunderx2t99")
       .Case("0x0af", "thunderx2t99")
       .Case("0xa1", "thunderxt88")
       .Case("0x0a1", "thunderxt88")
       .Default("generic");
   }
 
   if (Implementer == "0x46") { // Fujitsu Ltd.
     return StringSwitch<const char *>(Part)
       .Case("0x001", "a64fx")
       .Default("generic");
   }
 
   if (Implementer == "0x4e") { // NVIDIA Corporation
     return StringSwitch<const char *>(Part)
         .Case("0x004", "carmel")
         .Default("generic");
   }
 
   if (Implementer == "0x48") // HiSilicon Technologies, Inc.
     // The CPU part is a 3 digit hexadecimal number with a 0x prefix. The
     // values correspond to the "Part number" in the CP15/c0 register. The
     // contents are specified in the various processor manuals.
     return StringSwitch<const char *>(Part)
       .Case("0xd01", "tsv110")
       .Default("generic");
 
   if (Implementer == "0x51") // Qualcomm Technologies, Inc.
     // The CPU part is a 3 digit hexadecimal number with a 0x prefix. The
     // values correspond to the "Part number" in the CP15/c0 register. The
     // contents are specified in the various processor manuals.
     return StringSwitch<const char *>(Part)
         .Case("0x06f", "krait") // APQ8064
         .Case("0x201", "kryo")
         .Case("0x205", "kryo")
         .Case("0x211", "kryo")
         .Case("0x800", "cortex-a73") // Kryo 2xx Gold
         .Case("0x801", "cortex-a73") // Kryo 2xx Silver
         .Case("0x802", "cortex-a75") // Kryo 3xx Gold
         .Case("0x803", "cortex-a75") // Kryo 3xx Silver
         .Case("0x804", "cortex-a76") // Kryo 4xx Gold
         .Case("0x805", "cortex-a76") // Kryo 4xx/5xx Silver
         .Case("0xc00", "falkor")
         .Case("0xc01", "saphira")
         .Default("generic");
   if (Implementer == "0x53") { // Samsung Electronics Co., Ltd.
     // The Exynos chips have a convoluted ID scheme that doesn't seem to follow
     // any predictive pattern across variants and parts.
     unsigned Variant = 0, Part = 0;
 
     // Look for the CPU variant line, whose value is a 1 digit hexadecimal
     // number, corresponding to the Variant bits in the CP15/C0 register.
     for (auto I : Lines)
       if (I.consume_front("CPU variant"))
         I.ltrim("\t :").getAsInteger(0, Variant);
 
     // Look for the CPU part line, whose value is a 3 digit hexadecimal
     // number, corresponding to the PartNum bits in the CP15/C0 register.
     for (auto I : Lines)
       if (I.consume_front("CPU part"))
         I.ltrim("\t :").getAsInteger(0, Part);
 
     unsigned Exynos = (Variant << 12) | Part;
     switch (Exynos) {
     default:
       // Default by falling through to Exynos M3.
       LLVM_FALLTHROUGH;
     case 0x1002:
       return "exynos-m3";
     case 0x1003:
       return "exynos-m4";
     }
   }
 
+  if (Implementer == "0xc0") { // Ampere Computing
+    return StringSwitch<const char *>(Part)
+        .Case("0xac3", "ampere1")
+        .Default("generic");
+  }
+
   return "generic";
 }
 
 namespace {
 StringRef getCPUNameFromS390Model(unsigned int Id, bool HaveVectorSupport) {
   switch (Id) {
     case 2064:  // z900 not supported by LLVM
     case 2066:
     case 2084:  // z990 not supported by LLVM
     case 2086:
     case 2094:  // z9-109 not supported by LLVM
     case 2096:
       return "generic";
     case 2097:
     case 2098:
       return "z10";
     case 2817:
     case 2818:
       return "z196";
     case 2827:
     case 2828:
       return "zEC12";
     case 2964:
     case 2965:
       return HaveVectorSupport? "z13" : "zEC12";
     case 3906:
     case 3907:
       return HaveVectorSupport? "z14" : "zEC12";
     case 8561:
     case 8562:
       return HaveVectorSupport? "z15" : "zEC12";
     case 3931:
     case 3932:
     default:
       return HaveVectorSupport? "arch14" : "zEC12";
   }
 }
 } // end anonymous namespace
 
 StringRef sys::detail::getHostCPUNameForS390x(StringRef ProcCpuinfoContent) {
   // STIDP is a privileged operation, so use /proc/cpuinfo instead.
 
   // The "processor 0:" line comes after a fair amount of other information,
   // including a cache breakdown, but this should be plenty.
   SmallVector<StringRef, 32> Lines;
   ProcCpuinfoContent.split(Lines, "\n");
 
   // Look for the CPU features.
   SmallVector<StringRef, 32> CPUFeatures;
   for (unsigned I = 0, E = Lines.size(); I != E; ++I)
     if (Lines[I].startswith("features")) {
       size_t Pos = Lines[I].find(':');
       if (Pos != StringRef::npos) {
         Lines[I].drop_front(Pos + 1).split(CPUFeatures, ' ');
         break;
       }
     }
 
   // We need to check for the presence of vector support independently of
   // the machine type, since we may only use the vector register set when
   // supported by the kernel (and hypervisor).
   bool HaveVectorSupport = false;
   for (unsigned I = 0, E = CPUFeatures.size(); I != E; ++I) {
     if (CPUFeatures[I] == "vx")
       HaveVectorSupport = true;
   }
 
   // Now check the processor machine type.
   for (unsigned I = 0, E = Lines.size(); I != E; ++I) {
     if (Lines[I].startswith("processor ")) {
       size_t Pos = Lines[I].find("machine = ");
       if (Pos != StringRef::npos) {
         Pos += sizeof("machine = ") - 1;
         unsigned int Id;
         if (!Lines[I].drop_front(Pos).getAsInteger(10, Id))
           return getCPUNameFromS390Model(Id, HaveVectorSupport);
       }
       break;
     }
   }
 
   return "generic";
 }
 
 StringRef sys::detail::getHostCPUNameForBPF() {
 #if !defined(__linux__) || !defined(__x86_64__)
   return "generic";
 #else
   uint8_t v3_insns[40] __attribute__ ((aligned (8))) =
       /* BPF_MOV64_IMM(BPF_REG_0, 0) */
     { 0xb7, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
       /* BPF_MOV64_IMM(BPF_REG_2, 1) */
       0xb7, 0x2, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0,
       /* BPF_JMP32_REG(BPF_JLT, BPF_REG_0, BPF_REG_2, 1) */
       0xae, 0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0,
       /* BPF_MOV64_IMM(BPF_REG_0, 1) */
       0xb7, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0,
       /* BPF_EXIT_INSN() */
       0x95, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 };
 
   uint8_t v2_insns[40] __attribute__ ((aligned (8))) =
       /* BPF_MOV64_IMM(BPF_REG_0, 0) */
     { 0xb7, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
       /* BPF_MOV64_IMM(BPF_REG_2, 1) */
       0xb7, 0x2, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0,
       /* BPF_JMP_REG(BPF_JLT, BPF_REG_0, BPF_REG_2, 1) */
       0xad, 0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0,
       /* BPF_MOV64_IMM(BPF_REG_0, 1) */
       0xb7, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0,
       /* BPF_EXIT_INSN() */
       0x95, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 };
 
   struct bpf_prog_load_attr {
     uint32_t prog_type;
     uint32_t insn_cnt;
     uint64_t insns;
     uint64_t license;
     uint32_t log_level;
     uint32_t log_size;
     uint64_t log_buf;
     uint32_t kern_version;
     uint32_t prog_flags;
   } attr = {};
   attr.prog_type = 1; /* BPF_PROG_TYPE_SOCKET_FILTER */
   attr.insn_cnt = 5;
   attr.insns = (uint64_t)v3_insns;
   attr.license = (uint64_t)"DUMMY";
 
   int fd = syscall(321 /* __NR_bpf */, 5 /* BPF_PROG_LOAD */, &attr,
                    sizeof(attr));
   if (fd >= 0) {
     close(fd);
     return "v3";
   }
 
   /* Clear the whole attr in case its content changed by syscall. */
   memset(&attr, 0, sizeof(attr));
   attr.prog_type = 1; /* BPF_PROG_TYPE_SOCKET_FILTER */
   attr.insn_cnt = 5;
   attr.insns = (uint64_t)v2_insns;
   attr.license = (uint64_t)"DUMMY";
   fd = syscall(321 /* __NR_bpf */, 5 /* BPF_PROG_LOAD */, &attr, sizeof(attr));
   if (fd >= 0) {
     close(fd);
     return "v2";
   }
   return "v1";
 #endif
 }
 
 #if defined(__i386__) || defined(_M_IX86) || \
     defined(__x86_64__) || defined(_M_X64)
 
 // The check below for i386 was copied from clang's cpuid.h (__get_cpuid_max).
 // Check motivated by bug reports for OpenSSL crashing on CPUs without CPUID
 // support. Consequently, for i386, the presence of CPUID is checked first
 // via the corresponding eflags bit.
 // Removal of cpuid.h header motivated by PR30384
 // Header cpuid.h and method __get_cpuid_max are not used in llvm, clang, openmp
 // or test-suite, but are used in external projects e.g. libstdcxx
 static bool isCpuIdSupported() {
 #if defined(__GNUC__) || defined(__clang__)
 #if defined(__i386__)
   int __cpuid_supported;
   __asm__("  pushfl\n"
           "  popl   %%eax\n"
           "  movl   %%eax,%%ecx\n"
           "  xorl   $0x00200000,%%eax\n"
           "  pushl  %%eax\n"
           "  popfl\n"
           "  pushfl\n"
           "  popl   %%eax\n"
           "  movl   $0,%0\n"
           "  cmpl   %%eax,%%ecx\n"
           "  je     1f\n"
           "  movl   $1,%0\n"
           "1:"
           : "=r"(__cpuid_supported)
           :
           : "eax", "ecx");
   if (!__cpuid_supported)
     return false;
 #endif
   return true;
 #endif
   return true;
 }
 
 /// getX86CpuIDAndInfo - Execute the specified cpuid and return the 4 values in
 /// the specified arguments.  If we can't run cpuid on the host, return true.
 static bool getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX,
                                unsigned *rECX, unsigned *rEDX) {
 #if defined(__GNUC__) || defined(__clang__)
 #if defined(__x86_64__)
   // gcc doesn't know cpuid would clobber ebx/rbx. Preserve it manually.
   // FIXME: should we save this for Clang?
   __asm__("movq\t%%rbx, %%rsi\n\t"
           "cpuid\n\t"
           "xchgq\t%%rbx, %%rsi\n\t"
           : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
           : "a"(value));
   return false;
 #elif defined(__i386__)
   __asm__("movl\t%%ebx, %%esi\n\t"
           "cpuid\n\t"
           "xchgl\t%%ebx, %%esi\n\t"
           : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
           : "a"(value));
   return false;
 #else
   return true;
 #endif
 #elif defined(_MSC_VER)
   // The MSVC intrinsic is portable across x86 and x64.
   int registers[4];
   __cpuid(registers, value);
   *rEAX = registers[0];
   *rEBX = registers[1];
   *rECX = registers[2];
   *rEDX = registers[3];
   return false;
 #else
   return true;
 #endif
 }
 
 namespace llvm {
 namespace sys {
 namespace detail {
 namespace x86 {
 
 VendorSignatures getVendorSignature(unsigned *MaxLeaf) {
   unsigned EAX = 0, EBX = 0, ECX = 0, EDX = 0;
   if (MaxLeaf == nullptr)
     MaxLeaf = &EAX;
   else
     *MaxLeaf = 0;
 
   if (!isCpuIdSupported())
     return VendorSignatures::UNKNOWN;
 
   if (getX86CpuIDAndInfo(0, MaxLeaf, &EBX, &ECX, &EDX) || *MaxLeaf < 1)
     return VendorSignatures::UNKNOWN;
 
   // "Genu ineI ntel"
   if (EBX == 0x756e6547 && EDX == 0x49656e69 && ECX == 0x6c65746e)
     return VendorSignatures::GENUINE_INTEL;
 
   // "Auth enti cAMD"
   if (EBX == 0x68747541 && EDX == 0x69746e65 && ECX == 0x444d4163)
     return VendorSignatures::AUTHENTIC_AMD;
 
   return VendorSignatures::UNKNOWN;
 }
 
 } // namespace x86
 } // namespace detail
 } // namespace sys
 } // namespace llvm
 
 using namespace llvm::sys::detail::x86;
 
 /// getX86CpuIDAndInfoEx - Execute the specified cpuid with subleaf and return
 /// the 4 values in the specified arguments.  If we can't run cpuid on the host,
 /// return true.
 static bool getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf,
                                  unsigned *rEAX, unsigned *rEBX, unsigned *rECX,
                                  unsigned *rEDX) {
 #if defined(__GNUC__) || defined(__clang__)
 #if defined(__x86_64__)
   // gcc doesn't know cpuid would clobber ebx/rbx. Preserve it manually.
   // FIXME: should we save this for Clang?
   __asm__("movq\t%%rbx, %%rsi\n\t"
           "cpuid\n\t"
           "xchgq\t%%rbx, %%rsi\n\t"
           : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
           : "a"(value), "c"(subleaf));
   return false;
 #elif defined(__i386__)
   __asm__("movl\t%%ebx, %%esi\n\t"
           "cpuid\n\t"
           "xchgl\t%%ebx, %%esi\n\t"
           : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
           : "a"(value), "c"(subleaf));
   return false;
 #else
   return true;
 #endif
 #elif defined(_MSC_VER)
   int registers[4];
   __cpuidex(registers, value, subleaf);
   *rEAX = registers[0];
   *rEBX = registers[1];
   *rECX = registers[2];
   *rEDX = registers[3];
   return false;
 #else
   return true;
 #endif
 }
 
 // Read control register 0 (XCR0). Used to detect features such as AVX.
 static bool getX86XCR0(unsigned *rEAX, unsigned *rEDX) {
 #if defined(__GNUC__) || defined(__clang__)
   // Check xgetbv; this uses a .byte sequence instead of the instruction
   // directly because older assemblers do not include support for xgetbv and
   // there is no easy way to conditionally compile based on the assembler used.
   __asm__(".byte 0x0f, 0x01, 0xd0" : "=a"(*rEAX), "=d"(*rEDX) : "c"(0));
   return false;
 #elif defined(_MSC_FULL_VER) && defined(_XCR_XFEATURE_ENABLED_MASK)
   unsigned long long Result = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
   *rEAX = Result;
   *rEDX = Result >> 32;
   return false;
 #else
   return true;
 #endif
 }
 
 static void detectX86FamilyModel(unsigned EAX, unsigned *Family,
                                  unsigned *Model) {
   *Family = (EAX >> 8) & 0xf; // Bits 8 - 11
   *Model = (EAX >> 4) & 0xf;  // Bits 4 - 7
   if (*Family == 6 || *Family == 0xf) {
     if (*Family == 0xf)
       // Examine extended family ID if family ID is F.
       *Family += (EAX >> 20) & 0xff; // Bits 20 - 27
     // Examine extended model ID if family ID is 6 or F.
     *Model += ((EAX >> 16) & 0xf) << 4; // Bits 16 - 19
   }
 }
 
 static StringRef
 getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
                                 const unsigned *Features,
                                 unsigned *Type, unsigned *Subtype) {
   auto testFeature = [&](unsigned F) {
     return (Features[F / 32] & (1U << (F % 32))) != 0;
   };
 
   StringRef CPU;
 
   switch (Family) {
   case 3:
     CPU = "i386";
     break;
   case 4:
     CPU = "i486";
     break;
   case 5:
     if (testFeature(X86::FEATURE_MMX)) {
       CPU = "pentium-mmx";
       break;
     }
     CPU = "pentium";
     break;
   case 6:
     switch (Model) {
     case 0x0f: // Intel Core 2 Duo processor, Intel Core 2 Duo mobile
                // processor, Intel Core 2 Quad processor, Intel Core 2 Quad
                // mobile processor, Intel Core 2 Extreme processor, Intel
                // Pentium Dual-Core processor, Intel Xeon processor, model
                // 0Fh. All processors are manufactured using the 65 nm process.
     case 0x16: // Intel Celeron processor model 16h. All processors are
                // manufactured using the 65 nm process
       CPU = "core2";
       *Type = X86::INTEL_CORE2;
       break;
     case 0x17: // Intel Core 2 Extreme processor, Intel Xeon processor, model
                // 17h. All processors are manufactured using the 45 nm process.
                //
                // 45nm: Penryn , Wolfdale, Yorkfield (XE)
     case 0x1d: // Intel Xeon processor MP. All processors are manufactured using
                // the 45 nm process.
       CPU = "penryn";
       *Type = X86::INTEL_CORE2;
       break;
     case 0x1a: // Intel Core i7 processor and Intel Xeon processor. All
                // processors are manufactured using the 45 nm process.
     case 0x1e: // Intel(R) Core(TM) i7 CPU         870  @ 2.93GHz.
                // As found in a Summer 2010 model iMac.
     case 0x1f:
     case 0x2e:              // Nehalem EX
       CPU = "nehalem";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_NEHALEM;
       break;
     case 0x25: // Intel Core i7, laptop version.
     case 0x2c: // Intel Core i7 processor and Intel Xeon processor. All
                // processors are manufactured using the 32 nm process.
     case 0x2f: // Westmere EX
       CPU = "westmere";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_WESTMERE;
       break;
     case 0x2a: // Intel Core i7 processor. All processors are manufactured
                // using the 32 nm process.
     case 0x2d:
       CPU = "sandybridge";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_SANDYBRIDGE;
       break;
     case 0x3a:
     case 0x3e:              // Ivy Bridge EP
       CPU = "ivybridge";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_IVYBRIDGE;
       break;
 
     // Haswell:
     case 0x3c:
     case 0x3f:
     case 0x45:
     case 0x46:
       CPU = "haswell";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_HASWELL;
       break;
 
     // Broadwell:
     case 0x3d:
     case 0x47:
     case 0x4f:
     case 0x56:
       CPU = "broadwell";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_BROADWELL;
       break;
 
     // Skylake:
     case 0x4e:              // Skylake mobile
     case 0x5e:              // Skylake desktop
     case 0x8e:              // Kaby Lake mobile
     case 0x9e:              // Kaby Lake desktop
     case 0xa5:              // Comet Lake-H/S
     case 0xa6:              // Comet Lake-U
       CPU = "skylake";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_SKYLAKE;
       break;
 
     // Rocketlake:
     case 0xa7:
       CPU = "rocketlake";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_ROCKETLAKE;
       break;
 
     // Skylake Xeon:
     case 0x55:
       *Type = X86::INTEL_COREI7;
       if (testFeature(X86::FEATURE_AVX512BF16)) {
         CPU = "cooperlake";
         *Subtype = X86::INTEL_COREI7_COOPERLAKE;
       } else if (testFeature(X86::FEATURE_AVX512VNNI)) {
         CPU = "cascadelake";
         *Subtype = X86::INTEL_COREI7_CASCADELAKE;
       } else {
         CPU = "skylake-avx512";
         *Subtype = X86::INTEL_COREI7_SKYLAKE_AVX512;
       }
       break;
 
     // Cannonlake:
     case 0x66:
       CPU = "cannonlake";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_CANNONLAKE;
       break;
 
     // Icelake:
     case 0x7d:
     case 0x7e:
       CPU = "icelake-client";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_ICELAKE_CLIENT;
       break;
 
     // Tigerlake:
     case 0x8c:
     case 0x8d:
       CPU = "tigerlake";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_TIGERLAKE;
       break;
 
     // Alderlake:
     case 0x97:
     case 0x9a:
       CPU = "alderlake";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_ALDERLAKE;
       break;
 
     // Icelake Xeon:
     case 0x6a:
     case 0x6c:
       CPU = "icelake-server";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_ICELAKE_SERVER;
       break;
 
     // Sapphire Rapids:
     case 0x8f:
       CPU = "sapphirerapids";
       *Type = X86::INTEL_COREI7;
       *Subtype = X86::INTEL_COREI7_SAPPHIRERAPIDS;
       break;
 
     case 0x1c: // Most 45 nm Intel Atom processors
     case 0x26: // 45 nm Atom Lincroft
     case 0x27: // 32 nm Atom Medfield
     case 0x35: // 32 nm Atom Midview
     case 0x36: // 32 nm Atom Midview
       CPU = "bonnell";
       *Type = X86::INTEL_BONNELL;
       break;
 
     // Atom Silvermont codes from the Intel software optimization guide.
     case 0x37:
     case 0x4a:
     case 0x4d:
     case 0x5a:
     case 0x5d:
     case 0x4c: // really airmont
       CPU = "silvermont";
       *Type = X86::INTEL_SILVERMONT;
       break;
     // Goldmont:
     case 0x5c: // Apollo Lake
     case 0x5f: // Denverton
       CPU = "goldmont";
       *Type = X86::INTEL_GOLDMONT;
       break;
     case 0x7a:
       CPU = "goldmont-plus";
       *Type = X86::INTEL_GOLDMONT_PLUS;
       break;
     case 0x86:
       CPU = "tremont";
       *Type = X86::INTEL_TREMONT;
       break;
 
     // Xeon Phi (Knights Landing + Knights Mill):
     case 0x57:
       CPU = "knl";
       *Type = X86::INTEL_KNL;
       break;
     case 0x85:
       CPU = "knm";
       *Type = X86::INTEL_KNM;
       break;
 
     default: // Unknown family 6 CPU, try to guess.
       // Don't both with Type/Subtype here, they aren't used by the caller.
       // They're used above to keep the code in sync with compiler-rt.
       // TODO detect tigerlake host from model
       if (testFeature(X86::FEATURE_AVX512VP2INTERSECT)) {
         CPU = "tigerlake";
       } else if (testFeature(X86::FEATURE_AVX512VBMI2)) {
         CPU = "icelake-client";
       } else if (testFeature(X86::FEATURE_AVX512VBMI)) {
         CPU = "cannonlake";
       } else if (testFeature(X86::FEATURE_AVX512BF16)) {
         CPU = "cooperlake";
       } else if (testFeature(X86::FEATURE_AVX512VNNI)) {
         CPU = "cascadelake";
       } else if (testFeature(X86::FEATURE_AVX512VL)) {
         CPU = "skylake-avx512";
       } else if (testFeature(X86::FEATURE_AVX512ER)) {
         CPU = "knl";
       } else if (testFeature(X86::FEATURE_CLFLUSHOPT)) {
         if (testFeature(X86::FEATURE_SHA))
           CPU = "goldmont";
         else
           CPU = "skylake";
       } else if (testFeature(X86::FEATURE_ADX)) {
         CPU = "broadwell";
       } else if (testFeature(X86::FEATURE_AVX2)) {
         CPU = "haswell";
       } else if (testFeature(X86::FEATURE_AVX)) {
         CPU = "sandybridge";
       } else if (testFeature(X86::FEATURE_SSE4_2)) {
         if (testFeature(X86::FEATURE_MOVBE))
           CPU = "silvermont";
         else
           CPU = "nehalem";
       } else if (testFeature(X86::FEATURE_SSE4_1)) {
         CPU = "penryn";
       } else if (testFeature(X86::FEATURE_SSSE3)) {
         if (testFeature(X86::FEATURE_MOVBE))
           CPU = "bonnell";
         else
           CPU = "core2";
       } else if (testFeature(X86::FEATURE_64BIT)) {
         CPU = "core2";
       } else if (testFeature(X86::FEATURE_SSE3)) {
         CPU = "yonah";
       } else if (testFeature(X86::FEATURE_SSE2)) {
         CPU = "pentium-m";
       } else if (testFeature(X86::FEATURE_SSE)) {
         CPU = "pentium3";
       } else if (testFeature(X86::FEATURE_MMX)) {
         CPU = "pentium2";
       } else {
         CPU = "pentiumpro";
       }
       break;
     }
     break;
   case 15: {
     if (testFeature(X86::FEATURE_64BIT)) {
       CPU = "nocona";
       break;
     }
     if (testFeature(X86::FEATURE_SSE3)) {
       CPU = "prescott";
       break;
     }
     CPU = "pentium4";
     break;
   }
   default:
     break; // Unknown.
   }
 
   return CPU;
 }
 
 static StringRef
 getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
                               const unsigned *Features,
                               unsigned *Type, unsigned *Subtype) {
   auto testFeature = [&](unsigned F) {
     return (Features[F / 32] & (1U << (F % 32))) != 0;
   };
 
   StringRef CPU;
 
   switch (Family) {
   case 4:
     CPU = "i486";
     break;
   case 5:
     CPU = "pentium";
     switch (Model) {
     case 6:
     case 7:
       CPU = "k6";
       break;
     case 8:
       CPU = "k6-2";
       break;
     case 9:
     case 13:
       CPU = "k6-3";
       break;
     case 10:
       CPU = "geode";
       break;
     }
     break;
   case 6:
     if (testFeature(X86::FEATURE_SSE)) {
       CPU = "athlon-xp";
       break;
     }
     CPU = "athlon";
     break;
   case 15:
     if (testFeature(X86::FEATURE_SSE3)) {
       CPU = "k8-sse3";
       break;
     }
     CPU = "k8";
     break;
   case 16:
     CPU = "amdfam10";
     *Type = X86::AMDFAM10H; // "amdfam10"
     switch (Model) {
     case 2:
       *Subtype = X86::AMDFAM10H_BARCELONA;
       break;
     case 4:
       *Subtype = X86::AMDFAM10H_SHANGHAI;
       break;
     case 8:
       *Subtype = X86::AMDFAM10H_ISTANBUL;
       break;
     }
     break;
   case 20:
     CPU = "btver1";
     *Type = X86::AMD_BTVER1;
     break;
   case 21:
     CPU = "bdver1";
     *Type = X86::AMDFAM15H;
     if (Model >= 0x60 && Model <= 0x7f) {
       CPU = "bdver4";
       *Subtype = X86::AMDFAM15H_BDVER4;
       break; // 60h-7Fh: Excavator
     }
     if (Model >= 0x30 && Model <= 0x3f) {
       CPU = "bdver3";
       *Subtype = X86::AMDFAM15H_BDVER3;
       break; // 30h-3Fh: Steamroller
     }
     if ((Model >= 0x10 && Model <= 0x1f) || Model == 0x02) {
       CPU = "bdver2";
       *Subtype = X86::AMDFAM15H_BDVER2;
       break; // 02h, 10h-1Fh: Piledriver
     }
     if (Model <= 0x0f) {
       *Subtype = X86::AMDFAM15H_BDVER1;
       break; // 00h-0Fh: Bulldozer
     }
     break;
   case 22:
     CPU = "btver2";
     *Type = X86::AMD_BTVER2;
     break;
   case 23:
     CPU = "znver1";
     *Type = X86::AMDFAM17H;
     if ((Model >= 0x30 && Model <= 0x3f) || Model == 0x71) {
       CPU = "znver2";
       *Subtype = X86::AMDFAM17H_ZNVER2;
       break; // 30h-3fh, 71h: Zen2
     }
     if (Model <= 0x0f) {
       *Subtype = X86::AMDFAM17H_ZNVER1;
       break; // 00h-0Fh: Zen1
     }
     break;
   case 25:
     CPU = "znver3";
     *Type = X86::AMDFAM19H;
     if (Model <= 0x0f) {
       *Subtype = X86::AMDFAM19H_ZNVER3;
       break; // 00h-0Fh: Zen3
     }
     break;
   default:
     break; // Unknown AMD CPU.
   }
 
   return CPU;
 }
 
 static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
                                  unsigned *Features) {
   unsigned EAX, EBX;
 
   auto setFeature = [&](unsigned F) {
     Features[F / 32] |= 1U << (F % 32);
   };
 
   if ((EDX >> 15) & 1)
     setFeature(X86::FEATURE_CMOV);
   if ((EDX >> 23) & 1)
     setFeature(X86::FEATURE_MMX);
   if ((EDX >> 25) & 1)
     setFeature(X86::FEATURE_SSE);
   if ((EDX >> 26) & 1)
     setFeature(X86::FEATURE_SSE2);
 
   if ((ECX >> 0) & 1)
     setFeature(X86::FEATURE_SSE3);
   if ((ECX >> 1) & 1)
     setFeature(X86::FEATURE_PCLMUL);
   if ((ECX >> 9) & 1)
     setFeature(X86::FEATURE_SSSE3);
   if ((ECX >> 12) & 1)
     setFeature(X86::FEATURE_FMA);
   if ((ECX >> 19) & 1)
     setFeature(X86::FEATURE_SSE4_1);
   if ((ECX >> 20) & 1) {
     setFeature(X86::FEATURE_SSE4_2);
     setFeature(X86::FEATURE_CRC32);
   }
   if ((ECX >> 23) & 1)
     setFeature(X86::FEATURE_POPCNT);
   if ((ECX >> 25) & 1)
     setFeature(X86::FEATURE_AES);
 
   if ((ECX >> 22) & 1)
     setFeature(X86::FEATURE_MOVBE);
 
   // If CPUID indicates support for XSAVE, XRESTORE and AVX, and XGETBV
   // indicates that the AVX registers will be saved and restored on context
   // switch, then we have full AVX support.
   const unsigned AVXBits = (1 << 27) | (1 << 28);
   bool HasAVX = ((ECX & AVXBits) == AVXBits) && !getX86XCR0(&EAX, &EDX) &&
                 ((EAX & 0x6) == 0x6);
 #if defined(__APPLE__)
   // Darwin lazily saves the AVX512 context on first use: trust that the OS will
   // save the AVX512 context if we use AVX512 instructions, even the bit is not
   // set right now.
   bool HasAVX512Save = true;
 #else
   // AVX512 requires additional context to be saved by the OS.
   bool HasAVX512Save = HasAVX && ((EAX & 0xe0) == 0xe0);
 #endif
 
   if (HasAVX)
     setFeature(X86::FEATURE_AVX);
 
   bool HasLeaf7 =
       MaxLeaf >= 0x7 && !getX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX);
 
   if (HasLeaf7 && ((EBX >> 3) & 1))
     setFeature(X86::FEATURE_BMI);
   if (HasLeaf7 && ((EBX >> 5) & 1) && HasAVX)
     setFeature(X86::FEATURE_AVX2);
   if (HasLeaf7 && ((EBX >> 8) & 1))
     setFeature(X86::FEATURE_BMI2);
   if (HasLeaf7 && ((EBX >> 16) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX512F);
   if (HasLeaf7 && ((EBX >> 17) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX512DQ);
   if (HasLeaf7 && ((EBX >> 19) & 1))
     setFeature(X86::FEATURE_ADX);
   if (HasLeaf7 && ((EBX >> 21) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX512IFMA);
   if (HasLeaf7 && ((EBX >> 23) & 1))
     setFeature(X86::FEATURE_CLFLUSHOPT);
   if (HasLeaf7 && ((EBX >> 26) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX512PF);
   if (HasLeaf7 && ((EBX >> 27) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX512ER);
   if (HasLeaf7 && ((EBX >> 28) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX512CD);
   if (HasLeaf7 && ((EBX >> 29) & 1))
     setFeature(X86::FEATURE_SHA);
   if (HasLeaf7 && ((EBX >> 30) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX512BW);
   if (HasLeaf7 && ((EBX >> 31) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX512VL);
 
   if (HasLeaf7 && ((ECX >> 1) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX512VBMI);
   if (HasLeaf7 && ((ECX >> 6) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX512VBMI2);
   if (HasLeaf7 && ((ECX >> 8) & 1))
     setFeature(X86::FEATURE_GFNI);
   if (HasLeaf7 && ((ECX >> 10) & 1) && HasAVX)
     setFeature(X86::FEATURE_VPCLMULQDQ);
   if (HasLeaf7 && ((ECX >> 11) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX512VNNI);
   if (HasLeaf7 && ((ECX >> 12) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX512BITALG);
   if (HasLeaf7 && ((ECX >> 14) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX512VPOPCNTDQ);
 
   if (HasLeaf7 && ((EDX >> 2) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX5124VNNIW);
   if (HasLeaf7 && ((EDX >> 3) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX5124FMAPS);
   if (HasLeaf7 && ((EDX >> 8) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX512VP2INTERSECT);
 
   bool HasLeaf7Subleaf1 =
       MaxLeaf >= 7 && !getX86CpuIDAndInfoEx(0x7, 0x1, &EAX, &EBX, &ECX, &EDX);
   if (HasLeaf7Subleaf1 && ((EAX >> 5) & 1) && HasAVX512Save)
     setFeature(X86::FEATURE_AVX512BF16);
 
   unsigned MaxExtLevel;
   getX86CpuIDAndInfo(0x80000000, &MaxExtLevel, &EBX, &ECX, &EDX);
 
   bool HasExtLeaf1 = MaxExtLevel >= 0x80000001 &&
                      !getX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX);
   if (HasExtLeaf1 && ((ECX >> 6) & 1))
     setFeature(X86::FEATURE_SSE4_A);
   if (HasExtLeaf1 && ((ECX >> 11) & 1))
     setFeature(X86::FEATURE_XOP);
   if (HasExtLeaf1 && ((ECX >> 16) & 1))
     setFeature(X86::FEATURE_FMA4);
 
   if (HasExtLeaf1 && ((EDX >> 29) & 1))
     setFeature(X86::FEATURE_64BIT);
 }
 
 StringRef sys::getHostCPUName() {
   unsigned MaxLeaf = 0;
   const VendorSignatures Vendor = getVendorSignature(&MaxLeaf);
   if (Vendor == VendorSignatures::UNKNOWN)
     return "generic";
 
   unsigned EAX = 0, EBX = 0, ECX = 0, EDX = 0;
   getX86CpuIDAndInfo(0x1, &EAX, &EBX, &ECX, &EDX);
 
   unsigned Family = 0, Model = 0;
   unsigned Features[(X86::CPU_FEATURE_MAX + 31) / 32] = {0};
   detectX86FamilyModel(EAX, &Family, &Model);
   getAvailableFeatures(ECX, EDX, MaxLeaf, Features);
 
   // These aren't consumed in this file, but we try to keep some source code the
   // same or similar to compiler-rt.
   unsigned Type = 0;
   unsigned Subtype = 0;
 
   StringRef CPU;
 
   if (Vendor == VendorSignatures::GENUINE_INTEL) {
     CPU = getIntelProcessorTypeAndSubtype(Family, Model, Features, &Type,
                                           &Subtype);
   } else if (Vendor == VendorSignatures::AUTHENTIC_AMD) {
     CPU = getAMDProcessorTypeAndSubtype(Family, Model, Features, &Type,
                                         &Subtype);
   }
 
   if (!CPU.empty())
     return CPU;
 
   return "generic";
 }
 
 #elif defined(__APPLE__) && (defined(__ppc__) || defined(__powerpc__))
 StringRef sys::getHostCPUName() {
   host_basic_info_data_t hostInfo;
   mach_msg_type_number_t infoCount;
 
   infoCount = HOST_BASIC_INFO_COUNT;
   mach_port_t hostPort = mach_host_self();
   host_info(hostPort, HOST_BASIC_INFO, (host_info_t)&hostInfo,
             &infoCount);
   mach_port_deallocate(mach_task_self(), hostPort);
 
   if (hostInfo.cpu_type != CPU_TYPE_POWERPC)
     return "generic";
 
   switch (hostInfo.cpu_subtype) {
   case CPU_SUBTYPE_POWERPC_601:
     return "601";
   case CPU_SUBTYPE_POWERPC_602:
     return "602";
   case CPU_SUBTYPE_POWERPC_603:
     return "603";
   case CPU_SUBTYPE_POWERPC_603e:
     return "603e";
   case CPU_SUBTYPE_POWERPC_603ev:
     return "603ev";
   case CPU_SUBTYPE_POWERPC_604:
     return "604";
   case CPU_SUBTYPE_POWERPC_604e:
     return "604e";
   case CPU_SUBTYPE_POWERPC_620:
     return "620";
   case CPU_SUBTYPE_POWERPC_750:
     return "750";
   case CPU_SUBTYPE_POWERPC_7400:
     return "7400";
   case CPU_SUBTYPE_POWERPC_7450:
     return "7450";
   case CPU_SUBTYPE_POWERPC_970:
     return "970";
   default:;
   }
 
   return "generic";
 }
 #elif defined(__linux__) && (defined(__ppc__) || defined(__powerpc__))
 StringRef sys::getHostCPUName() {
   std::unique_ptr<llvm::MemoryBuffer> P = getProcCpuinfoContent();
   StringRef Content = P ? P->getBuffer() : "";
   return detail::getHostCPUNameForPowerPC(Content);
 }
 #elif defined(__linux__) && (defined(__arm__) || defined(__aarch64__))
 StringRef sys::getHostCPUName() {
   std::unique_ptr<llvm::MemoryBuffer> P = getProcCpuinfoContent();
   StringRef Content = P ? P->getBuffer() : "";
   return detail::getHostCPUNameForARM(Content);
 }
 #elif defined(__linux__) && defined(__s390x__)
 StringRef sys::getHostCPUName() {
   std::unique_ptr<llvm::MemoryBuffer> P = getProcCpuinfoContent();
   StringRef Content = P ? P->getBuffer() : "";
   return detail::getHostCPUNameForS390x(Content);
 }
 #elif defined(__MVS__)
 StringRef sys::getHostCPUName() {
   // Get pointer to Communications Vector Table (CVT).
   // The pointer is located at offset 16 of the Prefixed Save Area (PSA).
   // It is stored as 31 bit pointer and will be zero-extended to 64 bit.
   int *StartToCVTOffset = reinterpret_cast<int *>(0x10);
   // Since its stored as a 31-bit pointer, get the 4 bytes from the start
   // of address.
   int ReadValue = *StartToCVTOffset;
   // Explicitly clear the high order bit.
   ReadValue = (ReadValue & 0x7FFFFFFF);
   char *CVT = reinterpret_cast<char *>(ReadValue);
   // The model number is located in the CVT prefix at offset -6 and stored as
   // signless packed decimal.
   uint16_t Id = *(uint16_t *)&CVT[-6];
   // Convert number to integer.
   Id = decodePackedBCD<uint16_t>(Id, false);
   // Check for vector support. It's stored in field CVTFLAG5 (offset 244),
   // bit CVTVEF (X'80'). The facilities list is part of the PSA but the vector
   // extension can only be used if bit CVTVEF is on.
   bool HaveVectorSupport = CVT[244] & 0x80;
   return getCPUNameFromS390Model(Id, HaveVectorSupport);
 }
 #elif defined(__APPLE__) && defined(__aarch64__)
 StringRef sys::getHostCPUName() {
   return "cyclone";
 }
 #elif defined(__APPLE__) && defined(__arm__)
 StringRef sys::getHostCPUName() {
   host_basic_info_data_t hostInfo;
   mach_msg_type_number_t infoCount;
 
   infoCount = HOST_BASIC_INFO_COUNT;
   mach_port_t hostPort = mach_host_self();
   host_info(hostPort, HOST_BASIC_INFO, (host_info_t)&hostInfo,
             &infoCount);
   mach_port_deallocate(mach_task_self(), hostPort);
 
   if (hostInfo.cpu_type != CPU_TYPE_ARM) {
     assert(false && "CPUType not equal to ARM should not be possible on ARM");
     return "generic";
   }
   switch (hostInfo.cpu_subtype) {
     case CPU_SUBTYPE_ARM_V7S:
       return "swift";
     default:;
     }
 
   return "generic";
 }
 #elif defined(_AIX)
 StringRef sys::getHostCPUName() {
   switch (_system_configuration.implementation) {
   case POWER_4:
     if (_system_configuration.version == PV_4_3)
       return "970";
     return "pwr4";
   case POWER_5:
     if (_system_configuration.version == PV_5)
       return "pwr5";
     return "pwr5x";
   case POWER_6:
     if (_system_configuration.version == PV_6_Compat)
       return "pwr6";
     return "pwr6x";
   case POWER_7:
     return "pwr7";
   case POWER_8:
     return "pwr8";
   case POWER_9:
     return "pwr9";
 // TODO: simplify this once the macro is available in all OS levels.
 #ifdef POWER_10
   case POWER_10:
 #else
   case 0x40000:
 #endif
     return "pwr10";
   default:
     return "generic";
   }
 }
 #elif defined(__riscv)
 StringRef sys::getHostCPUName() {
 #if __riscv_xlen == 64
   return "generic-rv64";
 #elif __riscv_xlen == 32
   return "generic-rv32";
 #else
 #error "Unhandled value of __riscv_xlen"
 #endif
 }
 #else
 StringRef sys::getHostCPUName() { return "generic"; }
 namespace llvm {
 namespace sys {
 namespace detail {
 namespace x86 {
 
 VendorSignatures getVendorSignature(unsigned *MaxLeaf) {
   return VendorSignatures::UNKNOWN;
 }
 
 } // namespace x86
 } // namespace detail
 } // namespace sys
 } // namespace llvm
 #endif
 
 #if defined(__linux__) && (defined(__i386__) || defined(__x86_64__))
 // On Linux, the number of physical cores can be computed from /proc/cpuinfo,
 // using the number of unique physical/core id pairs. The following
 // implementation reads the /proc/cpuinfo format on an x86_64 system.
 int computeHostNumPhysicalCores() {
   // Enabled represents the number of physical id/core id pairs with at least
   // one processor id enabled by the CPU affinity mask.
   cpu_set_t Affinity, Enabled;
   if (sched_getaffinity(0, sizeof(Affinity), &Affinity) != 0)
     return -1;
   CPU_ZERO(&Enabled);
 
   // Read /proc/cpuinfo as a stream (until EOF reached). It cannot be
   // mmapped because it appears to have 0 size.
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Text =
       llvm::MemoryBuffer::getFileAsStream("/proc/cpuinfo");
   if (std::error_code EC = Text.getError()) {
     llvm::errs() << "Can't read "
                  << "/proc/cpuinfo: " << EC.message() << "\n";
     return -1;
   }
   SmallVector<StringRef, 8> strs;
   (*Text)->getBuffer().split(strs, "\n", /*MaxSplit=*/-1,
                              /*KeepEmpty=*/false);
   int CurProcessor = -1;
   int CurPhysicalId = -1;
   int CurSiblings = -1;
   int CurCoreId = -1;
   for (StringRef Line : strs) {
     std::pair<StringRef, StringRef> Data = Line.split(':');
     auto Name = Data.first.trim();
     auto Val = Data.second.trim();
     // These fields are available if the kernel is configured with CONFIG_SMP.
     if (Name == "processor")
       Val.getAsInteger(10, CurProcessor);
     else if (Name == "physical id")
       Val.getAsInteger(10, CurPhysicalId);
     else if (Name == "siblings")
       Val.getAsInteger(10, CurSiblings);
     else if (Name == "core id") {
       Val.getAsInteger(10, CurCoreId);
       // The processor id corresponds to an index into cpu_set_t.
       if (CPU_ISSET(CurProcessor, &Affinity))
         CPU_SET(CurPhysicalId * CurSiblings + CurCoreId, &Enabled);
     }
   }
   return CPU_COUNT(&Enabled);
 }
 #elif defined(__linux__) && defined(__powerpc__)
 int computeHostNumPhysicalCores() {
   cpu_set_t Affinity;
   if (sched_getaffinity(0, sizeof(Affinity), &Affinity) == 0)
     return CPU_COUNT(&Affinity);
 
   // The call to sched_getaffinity() may have failed because the Affinity
   // mask is too small for the number of CPU's on the system (i.e. the
   // system has more than 1024 CPUs). Allocate a mask large enough for
   // twice as many CPUs.
   cpu_set_t *DynAffinity;
   DynAffinity = CPU_ALLOC(2048);
   if (sched_getaffinity(0, CPU_ALLOC_SIZE(2048), DynAffinity) == 0) {
     int NumCPUs = CPU_COUNT(DynAffinity);
     CPU_FREE(DynAffinity);
     return NumCPUs;
   }
   return -1;
 }
 #elif defined(__linux__) && defined(__s390x__)
 int computeHostNumPhysicalCores() { return sysconf(_SC_NPROCESSORS_ONLN); }
 #elif defined(__APPLE__)
 #include <sys/param.h>
 #include <sys/sysctl.h>
 
 // Gets the number of *physical cores* on the machine.
 int computeHostNumPhysicalCores() {
   uint32_t count;
   size_t len = sizeof(count);
   sysctlbyname("hw.physicalcpu", &count, &len, NULL, 0);
   if (count < 1) {
     int nm[2];
     nm[0] = CTL_HW;
     nm[1] = HW_AVAILCPU;
     sysctl(nm, 2, &count, &len, NULL, 0);
     if (count < 1)
       return -1;
   }
   return count;
 }
 #elif defined(__MVS__)
 int computeHostNumPhysicalCores() {
   enum {
     // Byte offset of the pointer to the Communications Vector Table (CVT) in
     // the Prefixed Save Area (PSA). The table entry is a 31-bit pointer and
     // will be zero-extended to uintptr_t.
     FLCCVT = 16,
     // Byte offset of the pointer to the Common System Data Area (CSD) in the
     // CVT. The table entry is a 31-bit pointer and will be zero-extended to
     // uintptr_t.
     CVTCSD = 660,
     // Byte offset to the number of live CPs in the LPAR, stored as a signed
     // 32-bit value in the table.
     CSD_NUMBER_ONLINE_STANDARD_CPS = 264,
   };
   char *PSA = 0;
   char *CVT = reinterpret_cast<char *>(
       static_cast<uintptr_t>(reinterpret_cast<unsigned int &>(PSA[FLCCVT])));
   char *CSD = reinterpret_cast<char *>(
       static_cast<uintptr_t>(reinterpret_cast<unsigned int &>(CVT[CVTCSD])));
   return reinterpret_cast<int &>(CSD[CSD_NUMBER_ONLINE_STANDARD_CPS]);
 }
 #elif defined(_WIN32) && LLVM_ENABLE_THREADS != 0
 // Defined in llvm/lib/Support/Windows/Threading.inc
 int computeHostNumPhysicalCores();
 #else
 // On other systems, return -1 to indicate unknown.
 static int computeHostNumPhysicalCores() { return -1; }
 #endif
 
 int sys::getHostNumPhysicalCores() {
   static int NumCores = computeHostNumPhysicalCores();
   return NumCores;
 }
 
 #if defined(__i386__) || defined(_M_IX86) || \
     defined(__x86_64__) || defined(_M_X64)
 bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
   unsigned EAX = 0, EBX = 0, ECX = 0, EDX = 0;
   unsigned MaxLevel;
 
   if (getX86CpuIDAndInfo(0, &MaxLevel, &EBX, &ECX, &EDX) || MaxLevel < 1)
     return false;
 
   getX86CpuIDAndInfo(1, &EAX, &EBX, &ECX, &EDX);
 
   Features["cx8"]    = (EDX >>  8) & 1;
   Features["cmov"]   = (EDX >> 15) & 1;
   Features["mmx"]    = (EDX >> 23) & 1;
   Features["fxsr"]   = (EDX >> 24) & 1;
   Features["sse"]    = (EDX >> 25) & 1;
   Features["sse2"]   = (EDX >> 26) & 1;
 
   Features["sse3"]   = (ECX >>  0) & 1;
   Features["pclmul"] = (ECX >>  1) & 1;
   Features["ssse3"]  = (ECX >>  9) & 1;
   Features["cx16"]   = (ECX >> 13) & 1;
   Features["sse4.1"] = (ECX >> 19) & 1;
   Features["sse4.2"] = (ECX >> 20) & 1;
   Features["crc32"]  = Features["sse4.2"];
   Features["movbe"]  = (ECX >> 22) & 1;
   Features["popcnt"] = (ECX >> 23) & 1;
   Features["aes"]    = (ECX >> 25) & 1;
   Features["rdrnd"]  = (ECX >> 30) & 1;
 
   // If CPUID indicates support for XSAVE, XRESTORE and AVX, and XGETBV
   // indicates that the AVX registers will be saved and restored on context
   // switch, then we have full AVX support.
   bool HasXSave = ((ECX >> 27) & 1) && !getX86XCR0(&EAX, &EDX);
   bool HasAVXSave = HasXSave && ((ECX >> 28) & 1) && ((EAX & 0x6) == 0x6);
 #if defined(__APPLE__)
   // Darwin lazily saves the AVX512 context on first use: trust that the OS will
   // save the AVX512 context if we use AVX512 instructions, even the bit is not
   // set right now.
   bool HasAVX512Save = true;
 #else
   // AVX512 requires additional context to be saved by the OS.
   bool HasAVX512Save = HasAVXSave && ((EAX & 0xe0) == 0xe0);
 #endif
   // AMX requires additional context to be saved by the OS.
   const unsigned AMXBits = (1 << 17) | (1 << 18);
   bool HasAMXSave = HasXSave && ((EAX & AMXBits) == AMXBits);
 
   Features["avx"]   = HasAVXSave;
   Features["fma"]   = ((ECX >> 12) & 1) && HasAVXSave;
   // Only enable XSAVE if OS has enabled support for saving YMM state.
   Features["xsave"] = ((ECX >> 26) & 1) && HasAVXSave;
   Features["f16c"]  = ((ECX >> 29) & 1) && HasAVXSave;
 
   unsigned MaxExtLevel;
   getX86CpuIDAndInfo(0x80000000, &MaxExtLevel, &EBX, &ECX, &EDX);
 
   bool HasExtLeaf1 = MaxExtLevel >= 0x80000001 &&
                      !getX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX);
   Features["sahf"]   = HasExtLeaf1 && ((ECX >>  0) & 1);
   Features["lzcnt"]  = HasExtLeaf1 && ((ECX >>  5) & 1);
   Features["sse4a"]  = HasExtLeaf1 && ((ECX >>  6) & 1);
   Features["prfchw"] = HasExtLeaf1 && ((ECX >>  8) & 1);
   Features["xop"]    = HasExtLeaf1 && ((ECX >> 11) & 1) && HasAVXSave;
   Features["lwp"]    = HasExtLeaf1 && ((ECX >> 15) & 1);
   Features["fma4"]   = HasExtLeaf1 && ((ECX >> 16) & 1) && HasAVXSave;
   Features["tbm"]    = HasExtLeaf1 && ((ECX >> 21) & 1);
   Features["mwaitx"] = HasExtLeaf1 && ((ECX >> 29) & 1);
 
   Features["64bit"]  = HasExtLeaf1 && ((EDX >> 29) & 1);
 
   // Miscellaneous memory related features, detected by
   // using the 0x80000008 leaf of the CPUID instruction
   bool HasExtLeaf8 = MaxExtLevel >= 0x80000008 &&
                      !getX86CpuIDAndInfo(0x80000008, &EAX, &EBX, &ECX, &EDX);
   Features["clzero"]   = HasExtLeaf8 && ((EBX >> 0) & 1);
   Features["wbnoinvd"] = HasExtLeaf8 && ((EBX >> 9) & 1);
 
   bool HasLeaf7 =
       MaxLevel >= 7 && !getX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX);
 
   Features["fsgsbase"]   = HasLeaf7 && ((EBX >>  0) & 1);
   Features["sgx"]        = HasLeaf7 && ((EBX >>  2) & 1);
   Features["bmi"]        = HasLeaf7 && ((EBX >>  3) & 1);
   // AVX2 is only supported if we have the OS save support from AVX.
   Features["avx2"]       = HasLeaf7 && ((EBX >>  5) & 1) && HasAVXSave;
   Features["bmi2"]       = HasLeaf7 && ((EBX >>  8) & 1);
   Features["invpcid"]    = HasLeaf7 && ((EBX >> 10) & 1);
   Features["rtm"]        = HasLeaf7 && ((EBX >> 11) & 1);
   // AVX512 is only supported if the OS supports the context save for it.
   Features["avx512f"]    = HasLeaf7 && ((EBX >> 16) & 1) && HasAVX512Save;
   Features["avx512dq"]   = HasLeaf7 && ((EBX >> 17) & 1) && HasAVX512Save;
   Features["rdseed"]     = HasLeaf7 && ((EBX >> 18) & 1);
   Features["adx"]        = HasLeaf7 && ((EBX >> 19) & 1);
   Features["avx512ifma"] = HasLeaf7 && ((EBX >> 21) & 1) && HasAVX512Save;
   Features["clflushopt"] = HasLeaf7 && ((EBX >> 23) & 1);
   Features["clwb"]       = HasLeaf7 && ((EBX >> 24) & 1);
   Features["avx512pf"]   = HasLeaf7 && ((EBX >> 26) & 1) && HasAVX512Save;
   Features["avx512er"]   = HasLeaf7 && ((EBX >> 27) & 1) && HasAVX512Save;
   Features["avx512cd"]   = HasLeaf7 && ((EBX >> 28) & 1) && HasAVX512Save;
   Features["sha"]        = HasLeaf7 && ((EBX >> 29) & 1);
   Features["avx512bw"]   = HasLeaf7 && ((EBX >> 30) & 1) && HasAVX512Save;
   Features["avx512vl"]   = HasLeaf7 && ((EBX >> 31) & 1) && HasAVX512Save;
 
   Features["prefetchwt1"]     = HasLeaf7 && ((ECX >>  0) & 1);
   Features["avx512vbmi"]      = HasLeaf7 && ((ECX >>  1) & 1) && HasAVX512Save;
   Features["pku"]             = HasLeaf7 && ((ECX >>  4) & 1);
   Features["waitpkg"]         = HasLeaf7 && ((ECX >>  5) & 1);
   Features["avx512vbmi2"]     = HasLeaf7 && ((ECX >>  6) & 1) && HasAVX512Save;
   Features["shstk"]           = HasLeaf7 && ((ECX >>  7) & 1);
   Features["gfni"]            = HasLeaf7 && ((ECX >>  8) & 1);
   Features["vaes"]            = HasLeaf7 && ((ECX >>  9) & 1) && HasAVXSave;
   Features["vpclmulqdq"]      = HasLeaf7 && ((ECX >> 10) & 1) && HasAVXSave;
   Features["avx512vnni"]      = HasLeaf7 && ((ECX >> 11) & 1) && HasAVX512Save;
   Features["avx512bitalg"]    = HasLeaf7 && ((ECX >> 12) & 1) && HasAVX512Save;
   Features["avx512vpopcntdq"] = HasLeaf7 && ((ECX >> 14) & 1) && HasAVX512Save;
   Features["rdpid"]           = HasLeaf7 && ((ECX >> 22) & 1);
   Features["kl"]              = HasLeaf7 && ((ECX >> 23) & 1); // key locker
   Features["cldemote"]        = HasLeaf7 && ((ECX >> 25) & 1);
   Features["movdiri"]         = HasLeaf7 && ((ECX >> 27) & 1);
   Features["movdir64b"]       = HasLeaf7 && ((ECX >> 28) & 1);
   Features["enqcmd"]          = HasLeaf7 && ((ECX >> 29) & 1);
 
   Features["uintr"]           = HasLeaf7 && ((EDX >> 5) & 1);
   Features["avx512vp2intersect"] =
       HasLeaf7 && ((EDX >> 8) & 1) && HasAVX512Save;
   Features["serialize"]       = HasLeaf7 && ((EDX >> 14) & 1);
   Features["tsxldtrk"]        = HasLeaf7 && ((EDX >> 16) & 1);
   // There are two CPUID leafs which information associated with the pconfig
   // instruction:
   // EAX=0x7, ECX=0x0 indicates the availability of the instruction (via the 18th
   // bit of EDX), while the EAX=0x1b leaf returns information on the
   // availability of specific pconfig leafs.
   // The target feature here only refers to the the first of these two.
   // Users might need to check for the availability of specific pconfig
   // leaves using cpuid, since that information is ignored while
   // detecting features using the "-march=native" flag.
   // For more info, see X86 ISA docs.
   Features["pconfig"] = HasLeaf7 && ((EDX >> 18) & 1);
   Features["amx-bf16"]   = HasLeaf7 && ((EDX >> 22) & 1) && HasAMXSave;
   Features["avx512fp16"] = HasLeaf7 && ((EDX >> 23) & 1) && HasAVX512Save;
   Features["amx-tile"]   = HasLeaf7 && ((EDX >> 24) & 1) && HasAMXSave;
   Features["amx-int8"]   = HasLeaf7 && ((EDX >> 25) & 1) && HasAMXSave;
   bool HasLeaf7Subleaf1 =
       MaxLevel >= 7 && !getX86CpuIDAndInfoEx(0x7, 0x1, &EAX, &EBX, &ECX, &EDX);
   Features["avxvnni"]    = HasLeaf7Subleaf1 && ((EAX >> 4) & 1) && HasAVXSave;
   Features["avx512bf16"] = HasLeaf7Subleaf1 && ((EAX >> 5) & 1) && HasAVX512Save;
   Features["hreset"]     = HasLeaf7Subleaf1 && ((EAX >> 22) & 1);
 
   bool HasLeafD = MaxLevel >= 0xd &&
                   !getX86CpuIDAndInfoEx(0xd, 0x1, &EAX, &EBX, &ECX, &EDX);
 
   // Only enable XSAVE if OS has enabled support for saving YMM state.
   Features["xsaveopt"] = HasLeafD && ((EAX >> 0) & 1) && HasAVXSave;
   Features["xsavec"]   = HasLeafD && ((EAX >> 1) & 1) && HasAVXSave;
   Features["xsaves"]   = HasLeafD && ((EAX >> 3) & 1) && HasAVXSave;
 
   bool HasLeaf14 = MaxLevel >= 0x14 &&
                   !getX86CpuIDAndInfoEx(0x14, 0x0, &EAX, &EBX, &ECX, &EDX);
 
   Features["ptwrite"] = HasLeaf14 && ((EBX >> 4) & 1);
 
   bool HasLeaf19 =
       MaxLevel >= 0x19 && !getX86CpuIDAndInfo(0x19, &EAX, &EBX, &ECX, &EDX);
   Features["widekl"] = HasLeaf7 && HasLeaf19 && ((EBX >> 2) & 1);
 
   return true;
 }
 #elif defined(__linux__) && (defined(__arm__) || defined(__aarch64__))
 bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
   std::unique_ptr<llvm::MemoryBuffer> P = getProcCpuinfoContent();
   if (!P)
     return false;
 
   SmallVector<StringRef, 32> Lines;
   P->getBuffer().split(Lines, "\n");
 
   SmallVector<StringRef, 32> CPUFeatures;
 
   // Look for the CPU features.
   for (unsigned I = 0, E = Lines.size(); I != E; ++I)
     if (Lines[I].startswith("Features")) {
       Lines[I].split(CPUFeatures, ' ');
       break;
     }
 
 #if defined(__aarch64__)
   // Keep track of which crypto features we have seen
   enum { CAP_AES = 0x1, CAP_PMULL = 0x2, CAP_SHA1 = 0x4, CAP_SHA2 = 0x8 };
   uint32_t crypto = 0;
 #endif
 
   for (unsigned I = 0, E = CPUFeatures.size(); I != E; ++I) {
     StringRef LLVMFeatureStr = StringSwitch<StringRef>(CPUFeatures[I])
 #if defined(__aarch64__)
                                    .Case("asimd", "neon")
                                    .Case("fp", "fp-armv8")
                                    .Case("crc32", "crc")
 #else
                                    .Case("half", "fp16")
                                    .Case("neon", "neon")
                                    .Case("vfpv3", "vfp3")
                                    .Case("vfpv3d16", "d16")
                                    .Case("vfpv4", "vfp4")
                                    .Case("idiva", "hwdiv-arm")
                                    .Case("idivt", "hwdiv")
 #endif
                                    .Default("");
 
 #if defined(__aarch64__)
     // We need to check crypto separately since we need all of the crypto
     // extensions to enable the subtarget feature
     if (CPUFeatures[I] == "aes")
       crypto |= CAP_AES;
     else if (CPUFeatures[I] == "pmull")
       crypto |= CAP_PMULL;
     else if (CPUFeatures[I] == "sha1")
       crypto |= CAP_SHA1;
     else if (CPUFeatures[I] == "sha2")
       crypto |= CAP_SHA2;
 #endif
 
     if (LLVMFeatureStr != "")
       Features[LLVMFeatureStr] = true;
   }
 
 #if defined(__aarch64__)
   // If we have all crypto bits we can add the feature
   if (crypto == (CAP_AES | CAP_PMULL | CAP_SHA1 | CAP_SHA2))
     Features["crypto"] = true;
 #endif
 
   return true;
 }
 #elif defined(_WIN32) && (defined(__aarch64__) || defined(_M_ARM64))
 bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
   if (IsProcessorFeaturePresent(PF_ARM_NEON_INSTRUCTIONS_AVAILABLE))
     Features["neon"] = true;
   if (IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE))
     Features["crc"] = true;
   if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE))
     Features["crypto"] = true;
 
   return true;
 }
 #else
 bool sys::getHostCPUFeatures(StringMap<bool> &Features) { return false; }
 #endif
 
 std::string sys::getProcessTriple() {
   std::string TargetTripleString = updateTripleOSVersion(LLVM_HOST_TRIPLE);
   Triple PT(Triple::normalize(TargetTripleString));
 
   if (sizeof(void *) == 8 && PT.isArch32Bit())
     PT = PT.get64BitArchVariant();
   if (sizeof(void *) == 4 && PT.isArch64Bit())
     PT = PT.get32BitArchVariant();
 
   return PT.str();
 }
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64.td b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64.td
index 70c7b7b3f5dc..80e574b7b886 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64.td
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64.td
@@ -1,1239 +1,1257 @@
 //=- AArch64.td - Describe the AArch64 Target Machine --------*- tablegen -*-=//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //
 //===----------------------------------------------------------------------===//
 
 //===----------------------------------------------------------------------===//
 // Target-independent interfaces which we are implementing.
 //===----------------------------------------------------------------------===//
 
 include "llvm/Target/Target.td"
 
 //===----------------------------------------------------------------------===//
 // AArch64 Subtarget features.
 //
 
 def FeatureFPARMv8 : SubtargetFeature<"fp-armv8", "HasFPARMv8", "true",
                                        "Enable ARMv8 FP">;
 
 def FeatureNEON : SubtargetFeature<"neon", "HasNEON", "true",
   "Enable Advanced SIMD instructions", [FeatureFPARMv8]>;
 
 def FeatureSM4 : SubtargetFeature<
     "sm4", "HasSM4", "true",
     "Enable SM3 and SM4 support", [FeatureNEON]>;
 
 def FeatureSHA2 : SubtargetFeature<
     "sha2", "HasSHA2", "true",
     "Enable SHA1 and SHA256 support", [FeatureNEON]>;
 
 def FeatureSHA3 : SubtargetFeature<
     "sha3", "HasSHA3", "true",
     "Enable SHA512 and SHA3 support", [FeatureNEON, FeatureSHA2]>;
 
 def FeatureAES : SubtargetFeature<
     "aes", "HasAES", "true",
     "Enable AES support", [FeatureNEON]>;
 
 // Crypto has been split up and any combination is now valid (see the
 // crypto definitions above). Also, crypto is now context sensitive:
 // it has a different meaning for e.g. Armv8.4 than it has for Armv8.2.
 // Therefore, we rely on Clang, the user interacing tool, to pass on the
 // appropriate crypto options. But here in the backend, crypto has very little
 // meaning anymore. We kept the Crypto definition here for backward
 // compatibility, and now imply features SHA2 and AES, which was the
 // "traditional" meaning of Crypto.
 def FeatureCrypto : SubtargetFeature<"crypto", "HasCrypto", "true",
   "Enable cryptographic instructions", [FeatureNEON, FeatureSHA2, FeatureAES]>;
 
 def FeatureCRC : SubtargetFeature<"crc", "HasCRC", "true",
   "Enable ARMv8 CRC-32 checksum instructions">;
 
 def FeatureRAS : SubtargetFeature<"ras", "HasRAS", "true",
   "Enable ARMv8 Reliability, Availability and Serviceability Extensions">;
 
 def FeatureLSE : SubtargetFeature<"lse", "HasLSE", "true",
   "Enable ARMv8.1 Large System Extension (LSE) atomic instructions">;
 
 def FeatureLSE2 : SubtargetFeature<"lse2", "HasLSE2", "true",
   "Enable ARMv8.4 Large System Extension 2 (LSE2) atomicity rules">;
 
 def FeatureOutlineAtomics : SubtargetFeature<"outline-atomics", "OutlineAtomics", "true",
   "Enable out of line atomics to support LSE instructions">;
 
 def FeatureRDM : SubtargetFeature<"rdm", "HasRDM", "true",
   "Enable ARMv8.1 Rounding Double Multiply Add/Subtract instructions">;
 
 def FeaturePAN : SubtargetFeature<
     "pan", "HasPAN", "true",
     "Enables ARM v8.1 Privileged Access-Never extension">;
 
 def FeatureLOR : SubtargetFeature<
     "lor", "HasLOR", "true",
     "Enables ARM v8.1 Limited Ordering Regions extension">;
 
 def FeatureCONTEXTIDREL2 : SubtargetFeature<"CONTEXTIDREL2", "HasCONTEXTIDREL2",
     "true", "Enable RW operand CONTEXTIDR_EL2" >;
 
 def FeatureVH : SubtargetFeature<"vh", "HasVH", "true",
     "Enables ARM v8.1 Virtual Host extension", [FeatureCONTEXTIDREL2] >;
 
 def FeaturePerfMon : SubtargetFeature<"perfmon", "HasPerfMon", "true",
   "Enable ARMv8 PMUv3 Performance Monitors extension">;
 
 def FeatureFullFP16 : SubtargetFeature<"fullfp16", "HasFullFP16", "true",
   "Full FP16", [FeatureFPARMv8]>;
 
 def FeatureFP16FML : SubtargetFeature<"fp16fml", "HasFP16FML", "true",
   "Enable FP16 FML instructions", [FeatureFullFP16]>;
 
 def FeatureSPE : SubtargetFeature<"spe", "HasSPE", "true",
   "Enable Statistical Profiling extension">;
 
 def FeaturePAN_RWV : SubtargetFeature<
     "pan-rwv", "HasPAN_RWV", "true",
     "Enable v8.2 PAN s1e1R and s1e1W Variants",
     [FeaturePAN]>;
 
 // UAO PState
 def FeaturePsUAO : SubtargetFeature< "uaops", "HasPsUAO", "true",
     "Enable v8.2 UAO PState">;
 
 def FeatureCCPP : SubtargetFeature<"ccpp", "HasCCPP",
     "true", "Enable v8.2 data Cache Clean to Point of Persistence" >;
 
 def FeatureSVE : SubtargetFeature<"sve", "HasSVE", "true",
   "Enable Scalable Vector Extension (SVE) instructions", [FeatureFullFP16]>;
 
 // This flag is currently still labeled as Experimental, but when fully
 // implemented this should tell the compiler to use the zeroing pseudos to
 // benefit from the reverse instructions (e.g. SUB vs SUBR) if the inactive
 // lanes are known to be zero. The pseudos will then be expanded using the
 // MOVPRFX instruction to zero the inactive lanes. This feature should only be
 // enabled if MOVPRFX instructions are known to merge with the destructive
 // operations they prefix.
 //
 // This feature could similarly be extended to support cheap merging of _any_
 // value into the inactive lanes using the MOVPRFX instruction that uses
 // merging-predication.
 def FeatureExperimentalZeroingPseudos
     : SubtargetFeature<"use-experimental-zeroing-pseudos",
                        "UseExperimentalZeroingPseudos", "true",
                        "Hint to the compiler that the MOVPRFX instruction is "
                        "merged with destructive operations",
                        []>;
 
 def FeatureUseScalarIncVL : SubtargetFeature<"use-scalar-inc-vl",
   "UseScalarIncVL", "true", "Prefer inc/dec over add+cnt">;
 
 def FeatureSVE2 : SubtargetFeature<"sve2", "HasSVE2", "true",
   "Enable Scalable Vector Extension 2 (SVE2) instructions",
   [FeatureSVE, FeatureUseScalarIncVL]>;
 
 def FeatureSVE2AES : SubtargetFeature<"sve2-aes", "HasSVE2AES", "true",
   "Enable AES SVE2 instructions", [FeatureSVE2, FeatureAES]>;
 
 def FeatureSVE2SM4 : SubtargetFeature<"sve2-sm4", "HasSVE2SM4", "true",
   "Enable SM4 SVE2 instructions", [FeatureSVE2, FeatureSM4]>;
 
 def FeatureSVE2SHA3 : SubtargetFeature<"sve2-sha3", "HasSVE2SHA3", "true",
   "Enable SHA3 SVE2 instructions", [FeatureSVE2, FeatureSHA3]>;
 
 def FeatureSVE2BitPerm : SubtargetFeature<"sve2-bitperm", "HasSVE2BitPerm", "true",
   "Enable bit permutation SVE2 instructions", [FeatureSVE2]>;
 
 def FeatureZCRegMove : SubtargetFeature<"zcm", "HasZeroCycleRegMove", "true",
                                         "Has zero-cycle register moves">;
 
 def FeatureZCZeroingGP : SubtargetFeature<"zcz-gp", "HasZeroCycleZeroingGP", "true",
                                         "Has zero-cycle zeroing instructions for generic registers">;
 
 def FeatureNoZCZeroingFP : SubtargetFeature<"no-zcz-fp", "HasZeroCycleZeroingFP", "false",
                                         "Has no zero-cycle zeroing instructions for FP registers">;
 
 def FeatureZCZeroing : SubtargetFeature<"zcz", "HasZeroCycleZeroing", "true",
                                         "Has zero-cycle zeroing instructions",
                                         [FeatureZCZeroingGP]>;
 
 /// ... but the floating-point version doesn't quite work in rare cases on older
 /// CPUs.
 def FeatureZCZeroingFPWorkaround : SubtargetFeature<"zcz-fp-workaround",
     "HasZeroCycleZeroingFPWorkaround", "true",
     "The zero-cycle floating-point zeroing instruction has a bug">;
 
 def FeatureStrictAlign : SubtargetFeature<"strict-align",
                                           "StrictAlign", "true",
                                           "Disallow all unaligned memory "
                                           "access">;
 
 foreach i = {1-7,9-15,18,20-28,30} in
     def FeatureReserveX#i : SubtargetFeature<"reserve-x"#i, "ReserveXRegister["#i#"]", "true",
                                              "Reserve X"#i#", making it unavailable "
                                              "as a GPR">;
 
 foreach i = {8-15,18} in
     def FeatureCallSavedX#i : SubtargetFeature<"call-saved-x"#i,
          "CustomCallSavedXRegs["#i#"]", "true", "Make X"#i#" callee saved.">;
 
 def FeatureBalanceFPOps : SubtargetFeature<"balance-fp-ops", "BalanceFPOps",
     "true",
     "balance mix of odd and even D-registers for fp multiply(-accumulate) ops">;
 
 def FeaturePredictableSelectIsExpensive : SubtargetFeature<
     "predictable-select-expensive", "PredictableSelectIsExpensive", "true",
     "Prefer likely predicted branches over selects">;
 
 def FeatureCustomCheapAsMoveHandling : SubtargetFeature<"custom-cheap-as-move",
     "CustomAsCheapAsMove", "true",
     "Use custom handling of cheap instructions">;
 
 def FeatureExynosCheapAsMoveHandling : SubtargetFeature<"exynos-cheap-as-move",
     "ExynosAsCheapAsMove", "true",
     "Use Exynos specific handling of cheap instructions",
     [FeatureCustomCheapAsMoveHandling]>;
 
 def FeaturePostRAScheduler : SubtargetFeature<"use-postra-scheduler",
     "UsePostRAScheduler", "true", "Schedule again after register allocation">;
 
 def FeatureSlowMisaligned128Store : SubtargetFeature<"slow-misaligned-128store",
     "Misaligned128StoreIsSlow", "true", "Misaligned 128 bit stores are slow">;
 
 def FeatureSlowPaired128 : SubtargetFeature<"slow-paired-128",
     "Paired128IsSlow", "true", "Paired 128 bit loads and stores are slow">;
 
 def FeatureSlowSTRQro : SubtargetFeature<"slow-strqro-store", "STRQroIsSlow",
     "true", "STR of Q register with register offset is slow">;
 
 def FeatureAlternateSExtLoadCVTF32Pattern : SubtargetFeature<
     "alternate-sextload-cvt-f32-pattern", "UseAlternateSExtLoadCVTF32Pattern",
     "true", "Use alternative pattern for sextload convert to f32">;
 
 def FeatureArithmeticBccFusion : SubtargetFeature<
     "arith-bcc-fusion", "HasArithmeticBccFusion", "true",
     "CPU fuses arithmetic+bcc operations">;
 
 def FeatureArithmeticCbzFusion : SubtargetFeature<
     "arith-cbz-fusion", "HasArithmeticCbzFusion", "true",
     "CPU fuses arithmetic + cbz/cbnz operations">;
 
 def FeatureCmpBccFusion : SubtargetFeature<
     "cmp-bcc-fusion", "HasCmpBccFusion", "true",
     "CPU fuses cmp+bcc operations">;
 
 def FeatureFuseAddress : SubtargetFeature<
     "fuse-address", "HasFuseAddress", "true",
     "CPU fuses address generation and memory operations">;
 
 def FeatureFuseAES : SubtargetFeature<
     "fuse-aes", "HasFuseAES", "true",
     "CPU fuses AES crypto operations">;
 
 def FeatureFuseArithmeticLogic : SubtargetFeature<
     "fuse-arith-logic", "HasFuseArithmeticLogic", "true",
     "CPU fuses arithmetic and logic operations">;
 
 def FeatureFuseCCSelect : SubtargetFeature<
     "fuse-csel", "HasFuseCCSelect", "true",
     "CPU fuses conditional select operations">;
 
 def FeatureFuseCryptoEOR : SubtargetFeature<
     "fuse-crypto-eor", "HasFuseCryptoEOR", "true",
     "CPU fuses AES/PMULL and EOR operations">;
 
 def FeatureFuseLiterals : SubtargetFeature<
     "fuse-literals", "HasFuseLiterals", "true",
     "CPU fuses literal generation operations">;
 
 def FeatureDisableLatencySchedHeuristic : SubtargetFeature<
     "disable-latency-sched-heuristic", "DisableLatencySchedHeuristic", "true",
     "Disable latency scheduling heuristic">;
 
 def FeatureForce32BitJumpTables
    : SubtargetFeature<"force-32bit-jump-tables", "Force32BitJumpTables", "true",
                       "Force jump table entries to be 32-bits wide except at MinSize">;
 
 def FeatureRCPC : SubtargetFeature<"rcpc", "HasRCPC", "true",
                                    "Enable support for RCPC extension">;
 
 def FeatureUseRSqrt : SubtargetFeature<
     "use-reciprocal-square-root", "UseRSqrt", "true",
     "Use the reciprocal square root approximation">;
 
 def FeatureDotProd : SubtargetFeature<
     "dotprod", "HasDotProd", "true",
     "Enable dot product support">;
 
 def FeaturePAuth : SubtargetFeature<
     "pauth", "HasPAuth", "true",
     "Enable v8.3-A Pointer Authentication extension">;
 
 def FeatureJS : SubtargetFeature<
     "jsconv", "HasJS", "true",
     "Enable v8.3-A JavaScript FP conversion instructions",
     [FeatureFPARMv8]>;
 
 def FeatureCCIDX : SubtargetFeature<
     "ccidx", "HasCCIDX", "true",
     "Enable v8.3-A Extend of the CCSIDR number of sets">;
 
 def FeatureComplxNum : SubtargetFeature<
     "complxnum", "HasComplxNum", "true",
     "Enable v8.3-A Floating-point complex number support",
     [FeatureNEON]>;
 
 def FeatureNV : SubtargetFeature<
     "nv", "HasNV", "true",
     "Enable v8.4-A Nested Virtualization Enchancement">;
 
 def FeatureMPAM : SubtargetFeature<
     "mpam", "HasMPAM", "true",
     "Enable v8.4-A Memory system Partitioning and Monitoring extension">;
 
 def FeatureDIT : SubtargetFeature<
     "dit", "HasDIT", "true",
     "Enable v8.4-A Data Independent Timing instructions">;
 
 def FeatureTRACEV8_4 : SubtargetFeature<
     "tracev8.4", "HasTRACEV8_4", "true",
     "Enable v8.4-A Trace extension">;
 
 def FeatureAM : SubtargetFeature<
     "am", "HasAM", "true",
     "Enable v8.4-A Activity Monitors extension">;
 
 def FeatureAMVS : SubtargetFeature<
     "amvs", "HasAMVS", "true",
     "Enable v8.6-A Activity Monitors Virtualization support",
     [FeatureAM]>;
 
 def FeatureSEL2 : SubtargetFeature<
     "sel2", "HasSEL2", "true",
     "Enable v8.4-A Secure Exception Level 2 extension">;
 
 def FeatureTLB_RMI : SubtargetFeature<
     "tlb-rmi", "HasTLB_RMI", "true",
     "Enable v8.4-A TLB Range and Maintenance Instructions">;
 
 def FeatureFlagM : SubtargetFeature<
     "flagm", "HasFlagM", "true",
     "Enable v8.4-A Flag Manipulation Instructions">;
 
 // 8.4 RCPC enchancements: LDAPR & STLR instructions with Immediate Offset
 def FeatureRCPC_IMMO : SubtargetFeature<"rcpc-immo", "HasRCPC_IMMO", "true",
     "Enable v8.4-A RCPC instructions with Immediate Offsets",
     [FeatureRCPC]>;
 
 def FeatureNoNegativeImmediates : SubtargetFeature<"no-neg-immediates",
                                         "NegativeImmediates", "false",
                                         "Convert immediates and instructions "
                                         "to their negated or complemented "
                                         "equivalent when the immediate does "
                                         "not fit in the encoding.">;
 
 def FeatureLSLFast : SubtargetFeature<
     "lsl-fast", "HasLSLFast", "true",
     "CPU has a fastpath logical shift of up to 3 places">;
 
 def FeatureAggressiveFMA :
   SubtargetFeature<"aggressive-fma",
                    "HasAggressiveFMA",
                    "true",
                    "Enable Aggressive FMA for floating-point.">;
 
 def FeatureAltFPCmp : SubtargetFeature<"altnzcv", "HasAlternativeNZCV", "true",
   "Enable alternative NZCV format for floating point comparisons">;
 
 def FeatureFRInt3264 : SubtargetFeature<"fptoint", "HasFRInt3264", "true",
   "Enable FRInt[32|64][Z|X] instructions that round a floating-point number to "
   "an integer (in FP format) forcing it to fit into a 32- or 64-bit int" >;
 
 def FeatureSpecRestrict : SubtargetFeature<"specrestrict", "HasSpecRestrict",
   "true", "Enable architectural speculation restriction" >;
 
 def FeatureSB : SubtargetFeature<"sb", "HasSB",
   "true", "Enable v8.5 Speculation Barrier" >;
 
 def FeatureSSBS : SubtargetFeature<"ssbs", "HasSSBS",
   "true", "Enable Speculative Store Bypass Safe bit" >;
 
 def FeaturePredRes : SubtargetFeature<"predres", "HasPredRes", "true",
   "Enable v8.5a execution and data prediction invalidation instructions" >;
 
 def FeatureCacheDeepPersist : SubtargetFeature<"ccdp", "HasCCDP",
     "true", "Enable v8.5 Cache Clean to Point of Deep Persistence" >;
 
 def FeatureBranchTargetId : SubtargetFeature<"bti", "HasBTI",
     "true", "Enable Branch Target Identification" >;
 
 def FeatureRandGen : SubtargetFeature<"rand", "HasRandGen",
     "true", "Enable Random Number generation instructions" >;
 
 def FeatureMTE : SubtargetFeature<"mte", "HasMTE",
     "true", "Enable Memory Tagging Extension" >;
 
 def FeatureTRBE : SubtargetFeature<"trbe", "HasTRBE",
     "true", "Enable Trace Buffer Extension">;
 
 def FeatureETE : SubtargetFeature<"ete", "HasETE",
     "true", "Enable Embedded Trace Extension",
     [FeatureTRBE]>;
 
 def FeatureTME : SubtargetFeature<"tme", "HasTME",
     "true", "Enable Transactional Memory Extension" >;
 
 def FeatureTaggedGlobals : SubtargetFeature<"tagged-globals",
     "AllowTaggedGlobals",
     "true", "Use an instruction sequence for taking the address of a global "
     "that allows a memory tag in the upper address bits">;
 
 def FeatureBF16 : SubtargetFeature<"bf16", "HasBF16",
     "true", "Enable BFloat16 Extension" >;
 
 def FeatureMatMulInt8 : SubtargetFeature<"i8mm", "HasMatMulInt8",
     "true", "Enable Matrix Multiply Int8 Extension">;
 
 def FeatureMatMulFP32 : SubtargetFeature<"f32mm", "HasMatMulFP32",
     "true", "Enable Matrix Multiply FP32 Extension", [FeatureSVE]>;
 
 def FeatureMatMulFP64 : SubtargetFeature<"f64mm", "HasMatMulFP64",
     "true", "Enable Matrix Multiply FP64 Extension", [FeatureSVE]>;
 
 def FeatureXS : SubtargetFeature<"xs", "HasXS",
     "true", "Enable Armv8.7-A limited-TLB-maintenance instruction">;
 
 def FeatureWFxT : SubtargetFeature<"wfxt", "HasWFxT",
     "true", "Enable Armv8.7-A WFET and WFIT instruction">;
 
 def FeatureHCX : SubtargetFeature<
     "hcx", "HasHCX", "true", "Enable Armv8.7-A HCRX_EL2 system register">;
 
 def FeatureLS64 : SubtargetFeature<"ls64", "HasLS64",
     "true", "Enable Armv8.7-A LD64B/ST64B Accelerator Extension">;
 
 def FeatureHBC : SubtargetFeature<"hbc", "HasHBC",
     "true", "Enable Armv8.8-A Hinted Conditional Branches Extension">;
 
 def FeatureMOPS : SubtargetFeature<"mops", "HasMOPS",
     "true", "Enable Armv8.8-A memcpy and memset acceleration instructions">;
 
 def FeatureBRBE : SubtargetFeature<"brbe", "HasBRBE",
     "true", "Enable Branch Record Buffer Extension">;
 
 def FeatureSPE_EEF : SubtargetFeature<"spe-eef", "HasSPE_EEF",
     "true", "Enable extra register in the Statistical Profiling Extension">;
 
 def FeatureFineGrainedTraps : SubtargetFeature<"fgt", "HasFineGrainedTraps",
     "true", "Enable fine grained virtualization traps extension">;
 
 def FeatureEnhancedCounterVirtualization :
       SubtargetFeature<"ecv", "HasEnhancedCounterVirtualization",
       "true", "Enable enhanced counter virtualization extension">;
 
 def FeatureRME : SubtargetFeature<"rme", "HasRME",
     "true", "Enable Realm Management Extension">;
 
 // A subset of SVE(2) instructions are legal in Streaming SVE execution mode
 // defined by SME.
 def FeatureStreamingSVE : SubtargetFeature<"streaming-sve",
                                            "HasStreamingSVE", "true",
   "Enable subset of SVE(2) instructions for Streaming SVE execution mode">;
 def FeatureSME : SubtargetFeature<"sme", "HasSME", "true",
   "Enable Scalable Matrix Extension (SME)", [FeatureStreamingSVE, FeatureBF16]>;
 
 def FeatureSMEF64 : SubtargetFeature<"sme-f64", "HasSMEF64", "true",
   "Enable Scalable Matrix Extension (SME) F64F64 instructions", [FeatureSME]>;
 
 def FeatureSMEI64 : SubtargetFeature<"sme-i64", "HasSMEI64", "true",
   "Enable Scalable Matrix Extension (SME) I16I64 instructions", [FeatureSME]>;
 
 def FeatureAppleA7SysReg  : SubtargetFeature<"apple-a7-sysreg", "HasAppleA7SysReg", "true",
   "Apple A7 (the CPU formerly known as Cyclone)">;
 
 def FeatureEL2VMSA : SubtargetFeature<"el2vmsa", "HasEL2VMSA", "true",
   "Enable Exception Level 2 Virtual Memory System Architecture">;
 
 def FeatureEL3 : SubtargetFeature<"el3", "HasEL3", "true",
   "Enable Exception Level 3">;
 
 def FeatureFixCortexA53_835769 : SubtargetFeature<"fix-cortex-a53-835769",
   "FixCortexA53_835769", "true", "Mitigate Cortex-A53 Erratum 835769">;
 
 def FeatureNoBTIAtReturnTwice : SubtargetFeature<"no-bti-at-return-twice",
                                                  "NoBTIAtReturnTwice", "true",
                                                  "Don't place a BTI instruction "
                                                  "after a return-twice">;
 
 //===----------------------------------------------------------------------===//
 // Architectures.
 //
 def HasV8_0aOps : SubtargetFeature<"v8a", "HasV8_0aOps", "true",
   "Support ARM v8.0a instructions", [FeatureEL2VMSA, FeatureEL3]>;
 
 def HasV8_1aOps : SubtargetFeature<"v8.1a", "HasV8_1aOps", "true",
   "Support ARM v8.1a instructions", [HasV8_0aOps, FeatureCRC, FeatureLSE,
   FeatureRDM, FeaturePAN, FeatureLOR, FeatureVH]>;
 
 def HasV8_2aOps : SubtargetFeature<"v8.2a", "HasV8_2aOps", "true",
   "Support ARM v8.2a instructions", [HasV8_1aOps, FeaturePsUAO,
   FeaturePAN_RWV, FeatureRAS, FeatureCCPP]>;
 
 def HasV8_3aOps : SubtargetFeature<"v8.3a", "HasV8_3aOps", "true",
   "Support ARM v8.3a instructions", [HasV8_2aOps, FeatureRCPC, FeaturePAuth,
   FeatureJS, FeatureCCIDX, FeatureComplxNum]>;
 
 def HasV8_4aOps : SubtargetFeature<"v8.4a", "HasV8_4aOps", "true",
   "Support ARM v8.4a instructions", [HasV8_3aOps, FeatureDotProd,
   FeatureNV, FeatureMPAM, FeatureDIT,
   FeatureTRACEV8_4, FeatureAM, FeatureSEL2, FeatureTLB_RMI,
   FeatureFlagM, FeatureRCPC_IMMO, FeatureLSE2]>;
 
 def HasV8_5aOps : SubtargetFeature<
   "v8.5a", "HasV8_5aOps", "true", "Support ARM v8.5a instructions",
   [HasV8_4aOps, FeatureAltFPCmp, FeatureFRInt3264, FeatureSpecRestrict,
    FeatureSSBS, FeatureSB, FeaturePredRes, FeatureCacheDeepPersist,
    FeatureBranchTargetId]>;
 
 def HasV8_6aOps : SubtargetFeature<
   "v8.6a", "HasV8_6aOps", "true", "Support ARM v8.6a instructions",
   [HasV8_5aOps, FeatureAMVS, FeatureBF16, FeatureFineGrainedTraps,
    FeatureEnhancedCounterVirtualization, FeatureMatMulInt8]>;
 
 def HasV8_7aOps : SubtargetFeature<
   "v8.7a", "HasV8_7aOps", "true", "Support ARM v8.7a instructions",
   [HasV8_6aOps, FeatureXS, FeatureWFxT, FeatureHCX]>;
 
 def HasV8_8aOps : SubtargetFeature<
   "v8.8a", "HasV8_8aOps", "true", "Support ARM v8.8a instructions",
   [HasV8_7aOps, FeatureHBC, FeatureMOPS]>;
 
 def HasV9_0aOps : SubtargetFeature<
   "v9a", "HasV9_0aOps", "true", "Support ARM v9a instructions",
   [HasV8_5aOps, FeatureSVE2]>;
 
 def HasV9_1aOps : SubtargetFeature<
   "v9.1a", "HasV9_1aOps", "true", "Support ARM v9.1a instructions",
   [HasV8_6aOps, HasV9_0aOps]>;
 
 def HasV9_2aOps : SubtargetFeature<
   "v9.2a", "HasV9_2aOps", "true", "Support ARM v9.2a instructions",
   [HasV8_7aOps, HasV9_1aOps]>;
 
 def HasV9_3aOps : SubtargetFeature<
   "v9.3a", "HasV9_3aOps", "true", "Support ARM v9.3a instructions",
   [HasV8_8aOps, HasV9_2aOps]>;
 
 def HasV8_0rOps : SubtargetFeature<
   "v8r", "HasV8_0rOps", "true", "Support ARM v8r instructions",
   [//v8.1
   FeatureCRC, FeaturePAN, FeatureRDM, FeatureLSE, FeatureCONTEXTIDREL2,
   //v8.2
   FeatureRAS, FeaturePsUAO, FeatureCCPP, FeaturePAN_RWV,
   //v8.3
   FeatureComplxNum, FeatureCCIDX, FeatureJS,
   FeaturePAuth, FeatureRCPC,
   //v8.4
   FeatureDotProd, FeatureTRACEV8_4, FeatureTLB_RMI,
   FeatureFlagM, FeatureDIT, FeatureSEL2, FeatureRCPC_IMMO]>;
 
 //===----------------------------------------------------------------------===//
 // Register File Description
 //===----------------------------------------------------------------------===//
 
 include "AArch64RegisterInfo.td"
 include "AArch64RegisterBanks.td"
 include "AArch64CallingConvention.td"
 
 //===----------------------------------------------------------------------===//
 // Instruction Descriptions
 //===----------------------------------------------------------------------===//
 
 include "AArch64Schedule.td"
 include "AArch64InstrInfo.td"
 include "AArch64SchedPredicates.td"
 include "AArch64SchedPredExynos.td"
+include "AArch64SchedPredAmpere.td"
 include "AArch64Combine.td"
 
 def AArch64InstrInfo : InstrInfo;
 
 //===----------------------------------------------------------------------===//
 // Named operands for MRS/MSR/TLBI/...
 //===----------------------------------------------------------------------===//
 
 include "AArch64SystemOperands.td"
 
 //===----------------------------------------------------------------------===//
 // Access to privileged registers
 //===----------------------------------------------------------------------===//
 
 foreach i = 1-3 in
 def FeatureUseEL#i#ForTP : SubtargetFeature<"tpidr-el"#i, "UseEL"#i#"ForTP",
   "true", "Permit use of TPIDR_EL"#i#" for the TLS base">;
 
 //===----------------------------------------------------------------------===//
 // Control codegen mitigation against Straight Line Speculation vulnerability.
 //===----------------------------------------------------------------------===//
 
 def FeatureHardenSlsRetBr : SubtargetFeature<"harden-sls-retbr",
   "HardenSlsRetBr", "true",
   "Harden against straight line speculation across RET and BR instructions">;
 def FeatureHardenSlsBlr : SubtargetFeature<"harden-sls-blr",
   "HardenSlsBlr", "true",
   "Harden against straight line speculation across BLR instructions">;
 def FeatureHardenSlsNoComdat : SubtargetFeature<"harden-sls-nocomdat",
   "HardenSlsNoComdat", "true",
   "Generate thunk code for SLS mitigation in the normal text section">;
 
 //===----------------------------------------------------------------------===//
 // AArch64 Processors supported.
 //
 
 //===----------------------------------------------------------------------===//
 // Unsupported features to disable for scheduling models
 //===----------------------------------------------------------------------===//
 
 class AArch64Unsupported { list<Predicate> F; }
 
 def SVEUnsupported : AArch64Unsupported {
   let F = [HasSVE, HasSVE2, HasSVE2AES, HasSVE2SM4, HasSVE2SHA3,
            HasSVE2BitPerm, HasSVEorStreamingSVE, HasSVE2orStreamingSVE];
 }
 
 def PAUnsupported : AArch64Unsupported {
   let F = [HasPAuth];
 }
 
 def SMEUnsupported : AArch64Unsupported {
   let F = [HasSME, HasSMEF64, HasSMEI64];
 }
 
 include "AArch64SchedA53.td"
 include "AArch64SchedA55.td"
 include "AArch64SchedA57.td"
 include "AArch64SchedCyclone.td"
 include "AArch64SchedFalkor.td"
 include "AArch64SchedKryo.td"
 include "AArch64SchedExynosM3.td"
 include "AArch64SchedExynosM4.td"
 include "AArch64SchedExynosM5.td"
 include "AArch64SchedThunderX.td"
 include "AArch64SchedThunderX2T99.td"
 include "AArch64SchedA64FX.td"
 include "AArch64SchedThunderX3T110.td"
 include "AArch64SchedTSV110.td"
+include "AArch64SchedAmpere1.td"
 
 def TuneA35     : SubtargetFeature<"a35", "ARMProcFamily", "CortexA35",
                                 "Cortex-A35 ARM processors">;
 
 def TuneA53     : SubtargetFeature<"a53", "ARMProcFamily", "CortexA53",
                                    "Cortex-A53 ARM processors", [
                                    FeatureFuseAES,
                                    FeatureBalanceFPOps,
                                    FeatureCustomCheapAsMoveHandling,
                                    FeaturePostRAScheduler]>;
 
 def TuneA55     : SubtargetFeature<"a55", "ARMProcFamily", "CortexA55",
                                    "Cortex-A55 ARM processors", [
                                    FeatureFuseAES,
                                    FeaturePostRAScheduler,
                                    FeatureFuseAddress]>;
 
 def TuneA510    : SubtargetFeature<"a510", "ARMProcFamily", "CortexA510",
                                    "Cortex-A510 ARM processors", [
                                    FeatureFuseAES,
                                    FeaturePostRAScheduler
                                    ]>;
 
 def TuneA57     : SubtargetFeature<"a57", "ARMProcFamily", "CortexA57",
                                    "Cortex-A57 ARM processors", [
                                    FeatureFuseAES,
                                    FeatureBalanceFPOps,
                                    FeatureCustomCheapAsMoveHandling,
                                    FeatureFuseLiterals,
                                    FeaturePostRAScheduler,
                                    FeaturePredictableSelectIsExpensive]>;
 
 def TuneA65     : SubtargetFeature<"a65", "ARMProcFamily", "CortexA65",
                                    "Cortex-A65 ARM processors", [
                                    FeatureFuseAES,
                                    FeatureFuseAddress,
                                    FeatureFuseLiterals]>;
 
 def TuneA72     : SubtargetFeature<"a72", "ARMProcFamily", "CortexA72",
                                    "Cortex-A72 ARM processors", [
                                    FeatureFuseAES,
                                    FeatureFuseLiterals]>;
 
 def TuneA73     : SubtargetFeature<"a73", "ARMProcFamily", "CortexA73",
                                    "Cortex-A73 ARM processors", [
                                    FeatureFuseAES]>;
 
 def TuneA75     : SubtargetFeature<"a75", "ARMProcFamily", "CortexA75",
                                    "Cortex-A75 ARM processors", [
                                    FeatureFuseAES]>;
 
 def TuneA76     : SubtargetFeature<"a76", "ARMProcFamily", "CortexA76",
                                    "Cortex-A76 ARM processors", [
                                    FeatureFuseAES]>;
 
 def TuneA77     : SubtargetFeature<"a77", "ARMProcFamily", "CortexA77",
                                    "Cortex-A77 ARM processors", [
                                    FeatureCmpBccFusion,
                                    FeatureFuseAES]>;
 
 def TuneA78 : SubtargetFeature<"a78", "ARMProcFamily", "CortexA78",
                                "Cortex-A78 ARM processors", [
                                FeatureCmpBccFusion,
                                FeatureFuseAES,
                                FeaturePostRAScheduler]>;
 
 def TuneA78C : SubtargetFeature<"a78c", "ARMProcFamily",
                                 "CortexA78C",
                                 "Cortex-A78C ARM processors", [
                                 FeatureCmpBccFusion,
                                 FeatureFuseAES,
                                 FeaturePostRAScheduler]>;
 
 def TuneA710    : SubtargetFeature<"a710", "ARMProcFamily", "CortexA710",
                                    "Cortex-A710 ARM processors", [
                                    FeatureFuseAES,
                                    FeaturePostRAScheduler,
                                    FeatureCmpBccFusion]>;
 
 def TuneR82 : SubtargetFeature<"cortex-r82", "ARMProcFamily",
                                "CortexR82",
                                "Cortex-R82 ARM processors", [
                                FeaturePostRAScheduler]>;
 
 def TuneX1 : SubtargetFeature<"cortex-x1", "ARMProcFamily", "CortexX1",
                                   "Cortex-X1 ARM processors", [
                                   FeatureCmpBccFusion,
                                   FeatureFuseAES,
                                   FeaturePostRAScheduler]>;
 
 def TuneX2 : SubtargetFeature<"cortex-x2", "ARMProcFamily", "CortexX2",
                                   "Cortex-X2 ARM processors", [
                                   FeatureFuseAES,
                                   FeaturePostRAScheduler,
                                   FeatureCmpBccFusion]>;
 
 def TuneA64FX : SubtargetFeature<"a64fx", "ARMProcFamily", "A64FX",
                                  "Fujitsu A64FX processors", [
                                  FeaturePostRAScheduler,
                                  FeatureAggressiveFMA,
                                  FeatureArithmeticBccFusion,
                                  FeaturePredictableSelectIsExpensive
                                  ]>;
 
 def TuneCarmel : SubtargetFeature<"carmel", "ARMProcFamily", "Carmel",
                                   "Nvidia Carmel processors">;
 
 // Note that cyclone does not fuse AES instructions, but newer apple chips do
 // perform the fusion and cyclone is used by default when targetting apple OSes.
 def TuneAppleA7  : SubtargetFeature<"apple-a7", "ARMProcFamily", "AppleA7",
                                     "Apple A7 (the CPU formerly known as Cyclone)", [
                                     FeatureAlternateSExtLoadCVTF32Pattern,
                                     FeatureArithmeticBccFusion,
                                     FeatureArithmeticCbzFusion,
                                     FeatureDisableLatencySchedHeuristic,
                                     FeatureFuseAES, FeatureFuseCryptoEOR,
                                     FeatureZCRegMove,
                                     FeatureZCZeroing,
                                     FeatureZCZeroingFPWorkaround]
                                     >;
 
 def TuneAppleA10 : SubtargetFeature<"apple-a10", "ARMProcFamily", "AppleA10",
                                     "Apple A10", [
                                     FeatureAlternateSExtLoadCVTF32Pattern,
                                     FeatureArithmeticBccFusion,
                                     FeatureArithmeticCbzFusion,
                                     FeatureDisableLatencySchedHeuristic,
                                     FeatureFuseAES,
                                     FeatureFuseCryptoEOR,
                                     FeatureZCRegMove,
                                     FeatureZCZeroing]
                                     >;
 
 def TuneAppleA11 : SubtargetFeature<"apple-a11", "ARMProcFamily", "AppleA11",
                                     "Apple A11", [
                                     FeatureAlternateSExtLoadCVTF32Pattern,
                                     FeatureArithmeticBccFusion,
                                     FeatureArithmeticCbzFusion,
                                     FeatureDisableLatencySchedHeuristic,
                                     FeatureFuseAES,
                                     FeatureFuseCryptoEOR,
                                     FeatureZCRegMove,
                                     FeatureZCZeroing]
                                     >;
 
 def TuneAppleA12 : SubtargetFeature<"apple-a12", "ARMProcFamily", "AppleA12",
                                     "Apple A12", [
                                     FeatureAlternateSExtLoadCVTF32Pattern,
                                     FeatureArithmeticBccFusion,
                                     FeatureArithmeticCbzFusion,
                                     FeatureDisableLatencySchedHeuristic,
                                     FeatureFuseAES,
                                     FeatureFuseCryptoEOR,
                                     FeatureZCRegMove,
                                     FeatureZCZeroing]
                                     >;
 
 def TuneAppleA13 : SubtargetFeature<"apple-a13", "ARMProcFamily", "AppleA13",
                                     "Apple A13", [
                                     FeatureAlternateSExtLoadCVTF32Pattern,
                                     FeatureArithmeticBccFusion,
                                     FeatureArithmeticCbzFusion,
                                     FeatureDisableLatencySchedHeuristic,
                                     FeatureFuseAES,
                                     FeatureFuseCryptoEOR,
                                     FeatureZCRegMove,
                                     FeatureZCZeroing]
                                     >;
 
 def TuneAppleA14 : SubtargetFeature<"apple-a14", "ARMProcFamily", "AppleA14",
                                     "Apple A14", [
                                     FeatureAggressiveFMA,
                                     FeatureAlternateSExtLoadCVTF32Pattern,
                                     FeatureArithmeticBccFusion,
                                     FeatureArithmeticCbzFusion,
                                     FeatureDisableLatencySchedHeuristic,
                                     FeatureFuseAddress,
                                     FeatureFuseAES,
                                     FeatureFuseArithmeticLogic,
                                     FeatureFuseCCSelect,
                                     FeatureFuseCryptoEOR,
                                     FeatureFuseLiterals,
                                     FeatureZCRegMove,
                                     FeatureZCZeroing]>;
 
 def TuneExynosM3 : SubtargetFeature<"exynosm3", "ARMProcFamily", "ExynosM3",
                                     "Samsung Exynos-M3 processors",
                                     [FeatureExynosCheapAsMoveHandling,
                                      FeatureForce32BitJumpTables,
                                      FeatureFuseAddress,
                                      FeatureFuseAES,
                                      FeatureFuseCCSelect,
                                      FeatureFuseLiterals,
                                      FeatureLSLFast,
                                      FeaturePostRAScheduler,
                                      FeaturePredictableSelectIsExpensive]>;
 
 def TuneExynosM4 : SubtargetFeature<"exynosm3", "ARMProcFamily", "ExynosM3",
                                     "Samsung Exynos-M3 processors",
                                     [FeatureArithmeticBccFusion,
                                      FeatureArithmeticCbzFusion,
                                      FeatureExynosCheapAsMoveHandling,
                                      FeatureForce32BitJumpTables,
                                      FeatureFuseAddress,
                                      FeatureFuseAES,
                                      FeatureFuseArithmeticLogic,
                                      FeatureFuseCCSelect,
                                      FeatureFuseLiterals,
                                      FeatureLSLFast,
                                      FeaturePostRAScheduler,
                                      FeatureZCZeroing]>;
 
 def TuneKryo    : SubtargetFeature<"kryo", "ARMProcFamily", "Kryo",
                                    "Qualcomm Kryo processors", [
                                    FeatureCustomCheapAsMoveHandling,
                                    FeaturePostRAScheduler,
                                    FeaturePredictableSelectIsExpensive,
                                    FeatureZCZeroing,
                                    FeatureLSLFast]
                                    >;
 
 def TuneFalkor  : SubtargetFeature<"falkor", "ARMProcFamily", "Falkor",
                                    "Qualcomm Falkor processors", [
                                    FeatureCustomCheapAsMoveHandling,
                                    FeaturePostRAScheduler,
                                    FeaturePredictableSelectIsExpensive,
                                    FeatureZCZeroing,
                                    FeatureLSLFast,
                                    FeatureSlowSTRQro
                                    ]>;
 
 def TuneNeoverseE1 : SubtargetFeature<"neoversee1", "ARMProcFamily", "NeoverseE1",
                                       "Neoverse E1 ARM processors", [
                                       FeaturePostRAScheduler,
                                       FeatureFuseAES
                                       ]>;
 
 def TuneNeoverseN1 : SubtargetFeature<"neoversen1", "ARMProcFamily", "NeoverseN1",
                                       "Neoverse N1 ARM processors", [
                                       FeaturePostRAScheduler,
                                       FeatureFuseAES
                                       ]>;
 
 def TuneNeoverseN2 : SubtargetFeature<"neoversen2", "ARMProcFamily", "NeoverseN2",
                                       "Neoverse N2 ARM processors", [
                                       FeaturePostRAScheduler,
                                       FeatureFuseAES
                                       ]>;
 def TuneNeoverse512TVB : SubtargetFeature<"neoverse512tvb", "ARMProcFamily", "Neoverse512TVB",
                                       "Neoverse 512-TVB ARM processors", [
                                       FeaturePostRAScheduler,
                                       FeatureFuseAES
                                       ]>;
 
 def TuneNeoverseV1 : SubtargetFeature<"neoversev1", "ARMProcFamily", "NeoverseV1",
                                       "Neoverse V1 ARM processors", [
                                       FeatureFuseAES,
                                       FeaturePostRAScheduler]>;
 
 def TuneSaphira  : SubtargetFeature<"saphira", "ARMProcFamily", "Saphira",
                                    "Qualcomm Saphira processors", [
                                    FeatureCustomCheapAsMoveHandling,
                                    FeaturePostRAScheduler,
                                    FeaturePredictableSelectIsExpensive,
                                    FeatureZCZeroing,
                                    FeatureLSLFast]>;
 
 def TuneThunderX2T99  : SubtargetFeature<"thunderx2t99", "ARMProcFamily", "ThunderX2T99",
                                          "Cavium ThunderX2 processors", [
                                           FeatureAggressiveFMA,
                                           FeatureArithmeticBccFusion,
                                           FeaturePostRAScheduler,
                                           FeaturePredictableSelectIsExpensive]>;
 
 def TuneThunderX3T110  : SubtargetFeature<"thunderx3t110", "ARMProcFamily",
                                           "ThunderX3T110",
                                           "Marvell ThunderX3 processors", [
                                            FeatureAggressiveFMA,
                                            FeatureArithmeticBccFusion,
                                            FeaturePostRAScheduler,
                                            FeaturePredictableSelectIsExpensive,
                                            FeatureBalanceFPOps,
                                            FeatureStrictAlign]>;
 
 def TuneThunderX : SubtargetFeature<"thunderx", "ARMProcFamily", "ThunderX",
                                     "Cavium ThunderX processors", [
                                     FeaturePostRAScheduler,
                                     FeaturePredictableSelectIsExpensive]>;
 
 def TuneThunderXT88 : SubtargetFeature<"thunderxt88", "ARMProcFamily",
                                        "ThunderXT88",
                                        "Cavium ThunderX processors", [
                                        FeaturePostRAScheduler,
                                        FeaturePredictableSelectIsExpensive]>;
 
 def TuneThunderXT81 : SubtargetFeature<"thunderxt81", "ARMProcFamily",
                                        "ThunderXT81",
                                        "Cavium ThunderX processors", [
                                        FeaturePostRAScheduler,
                                        FeaturePredictableSelectIsExpensive]>;
 
 def TuneThunderXT83 : SubtargetFeature<"thunderxt83", "ARMProcFamily",
                                        "ThunderXT83",
                                        "Cavium ThunderX processors", [
                                        FeaturePostRAScheduler,
                                        FeaturePredictableSelectIsExpensive]>;
 
 def TuneTSV110 : SubtargetFeature<"tsv110", "ARMProcFamily", "TSV110",
                                   "HiSilicon TS-V110 processors", [
                                   FeatureCustomCheapAsMoveHandling,
                                   FeatureFuseAES,
                                   FeaturePostRAScheduler]>;
 
+def TuneAmpere1 : SubtargetFeature<"ampere1", "ARMProcFamily", "Ampere1",
+                                   "Ampere Computing Ampere-1 processors", [
+                                   FeaturePostRAScheduler,
+                                   FeatureFuseAES,
+                                   FeatureLSLFast,
+                                   FeatureAggressiveFMA,
+                                   FeatureArithmeticBccFusion,
+                                   FeatureCmpBccFusion,
+                                   FeatureFuseAddress,
+                                   FeatureFuseLiterals]>;
 
 def ProcessorFeatures {
   list<SubtargetFeature> A53  = [HasV8_0aOps, FeatureCRC, FeatureCrypto,
                                  FeatureFPARMv8, FeatureNEON, FeaturePerfMon];
   list<SubtargetFeature> A55  = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
                                  FeatureNEON, FeatureFullFP16, FeatureDotProd,
                                  FeatureRCPC, FeaturePerfMon];
   list<SubtargetFeature> A510 = [HasV9_0aOps, FeatureNEON, FeaturePerfMon,
                                  FeatureMatMulInt8, FeatureBF16, FeatureAM,
                                  FeatureMTE, FeatureETE, FeatureSVE2BitPerm,
                                  FeatureFP16FML];
   list<SubtargetFeature> A65  = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
                                  FeatureNEON, FeatureFullFP16, FeatureDotProd,
                                  FeatureRCPC, FeatureSSBS, FeatureRAS];
   list<SubtargetFeature> A76  = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
                                  FeatureNEON, FeatureFullFP16, FeatureDotProd,
                                  FeatureRCPC, FeatureSSBS];
   list<SubtargetFeature> A77  = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
                                  FeatureNEON, FeatureFullFP16, FeatureDotProd,
                                  FeatureRCPC, FeatureSSBS];
   list<SubtargetFeature> A78  = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
                                  FeatureNEON, FeatureFullFP16, FeatureDotProd,
                                  FeatureRCPC, FeaturePerfMon, FeatureSPE,
                                  FeatureSSBS];
   list<SubtargetFeature> A78C = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
                                  FeatureNEON, FeatureFullFP16, FeatureDotProd,
                                  FeatureFlagM, FeatureFP16FML, FeaturePAuth,
                                  FeaturePerfMon, FeatureRCPC, FeatureSPE,
                                  FeatureSSBS];
   list<SubtargetFeature> A710 = [HasV9_0aOps, FeatureNEON, FeaturePerfMon,
                                  FeatureETE, FeatureMTE, FeatureFP16FML,
                                  FeatureSVE2BitPerm, FeatureBF16, FeatureMatMulInt8];
   list<SubtargetFeature> R82  = [HasV8_0rOps, FeaturePerfMon, FeatureFullFP16,
                                  FeatureFP16FML, FeatureSSBS, FeaturePredRes,
                                  FeatureSB, FeatureSpecRestrict];
   list<SubtargetFeature> X1   = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
                                  FeatureNEON, FeatureRCPC, FeaturePerfMon,
                                  FeatureSPE, FeatureFullFP16, FeatureDotProd,
                                  FeatureSSBS];
   list<SubtargetFeature> X1C  = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
                                  FeatureNEON, FeatureRCPC, FeaturePerfMon,
                                  FeatureSPE, FeatureFullFP16, FeatureDotProd,
                                  FeaturePAuth, FeatureSSBS];
   list<SubtargetFeature> X2   = [HasV9_0aOps, FeatureNEON, FeaturePerfMon,
                                  FeatureMatMulInt8, FeatureBF16, FeatureAM,
                                  FeatureMTE, FeatureETE, FeatureSVE2BitPerm,
                                  FeatureFP16FML];
   list<SubtargetFeature> A64FX    = [HasV8_2aOps, FeatureFPARMv8, FeatureNEON,
                                      FeatureSHA2, FeaturePerfMon, FeatureFullFP16,
                                      FeatureSVE, FeatureComplxNum];
   list<SubtargetFeature> Carmel   = [HasV8_2aOps, FeatureNEON, FeatureCrypto,
                                      FeatureFullFP16];
   list<SubtargetFeature> AppleA7  = [HasV8_0aOps, FeatureCrypto, FeatureFPARMv8,
                                      FeatureNEON,FeaturePerfMon, FeatureAppleA7SysReg];
   list<SubtargetFeature> AppleA10 = [HasV8_0aOps, FeatureCrypto, FeatureFPARMv8,
                                      FeatureNEON, FeaturePerfMon, FeatureCRC,
                                      FeatureRDM, FeaturePAN, FeatureLOR, FeatureVH];
   list<SubtargetFeature> AppleA11 = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
                                      FeatureNEON, FeaturePerfMon, FeatureFullFP16];
   list<SubtargetFeature> AppleA12 = [HasV8_3aOps, FeatureCrypto, FeatureFPARMv8,
                                      FeatureNEON, FeaturePerfMon, FeatureFullFP16];
   list<SubtargetFeature> AppleA13 = [HasV8_4aOps, FeatureCrypto, FeatureFPARMv8,
                                      FeatureNEON, FeaturePerfMon, FeatureFullFP16,
                                      FeatureFP16FML, FeatureSHA3];
   list<SubtargetFeature> AppleA14 = [HasV8_4aOps, FeatureCrypto, FeatureFPARMv8,
                                      FeatureNEON, FeaturePerfMon, FeatureFRInt3264,
                                      FeatureSpecRestrict, FeatureSSBS, FeatureSB,
                                      FeaturePredRes, FeatureCacheDeepPersist,
                                      FeatureFullFP16, FeatureFP16FML, FeatureSHA3,
                                      FeatureAltFPCmp];
   list<SubtargetFeature> ExynosM3 = [HasV8_0aOps, FeatureCRC, FeatureCrypto,
                                      FeaturePerfMon];
   list<SubtargetFeature> ExynosM4 = [HasV8_2aOps, FeatureCrypto, FeatureDotProd,
                                      FeatureFullFP16, FeaturePerfMon];
   list<SubtargetFeature> Falkor   = [HasV8_0aOps, FeatureCRC, FeatureCrypto,
                                      FeatureFPARMv8, FeatureNEON, FeaturePerfMon,
                                      FeatureRDM];
   list<SubtargetFeature> NeoverseE1 = [HasV8_2aOps, FeatureCrypto, FeatureDotProd,
                                        FeatureFPARMv8, FeatureFullFP16, FeatureNEON,
                                        FeatureRCPC, FeatureSSBS];
   list<SubtargetFeature> NeoverseN1 = [HasV8_2aOps, FeatureCrypto, FeatureDotProd,
                                        FeatureFPARMv8, FeatureFullFP16, FeatureNEON,
                                        FeatureRCPC, FeatureSPE, FeatureSSBS];
   list<SubtargetFeature> NeoverseN2 = [HasV8_5aOps, FeatureBF16, FeatureETE,
                                        FeatureMatMulInt8, FeatureMTE, FeatureSVE2,
                                        FeatureSVE2BitPerm, FeatureTRBE, FeatureCrypto];
   list<SubtargetFeature> Neoverse512TVB = [HasV8_4aOps, FeatureBF16, FeatureCacheDeepPersist,
                                            FeatureCrypto, FeatureFPARMv8, FeatureFP16FML,
                                            FeatureFullFP16, FeatureMatMulInt8, FeatureNEON,
                                            FeaturePerfMon, FeatureRandGen, FeatureSPE,
                                            FeatureSSBS, FeatureSVE];
   list<SubtargetFeature> NeoverseV1 = [HasV8_4aOps, FeatureBF16, FeatureCacheDeepPersist,
                                        FeatureCrypto, FeatureFPARMv8, FeatureFP16FML,
                                        FeatureFullFP16, FeatureMatMulInt8, FeatureNEON,
                                        FeaturePerfMon, FeatureRandGen, FeatureSPE,
                                        FeatureSSBS, FeatureSVE];
   list<SubtargetFeature> Saphira    = [HasV8_4aOps, FeatureCrypto, FeatureFPARMv8,
                                        FeatureNEON, FeatureSPE, FeaturePerfMon];
   list<SubtargetFeature> ThunderX   = [HasV8_0aOps, FeatureCRC, FeatureCrypto,
                                        FeatureFPARMv8, FeaturePerfMon, FeatureNEON];
   list<SubtargetFeature> ThunderX2T99  = [HasV8_1aOps, FeatureCRC, FeatureCrypto,
                                           FeatureFPARMv8, FeatureNEON, FeatureLSE];
   list<SubtargetFeature> ThunderX3T110 = [HasV8_3aOps, FeatureCRC, FeatureCrypto,
                                           FeatureFPARMv8, FeatureNEON, FeatureLSE,
                                           FeaturePAuth, FeaturePerfMon];
   list<SubtargetFeature> TSV110 = [HasV8_2aOps, FeatureCrypto, FeatureFPARMv8,
                                    FeatureNEON, FeaturePerfMon, FeatureSPE,
                                    FeatureFullFP16, FeatureFP16FML, FeatureDotProd];
+  list<SubtargetFeature> Ampere1 = [HasV8_6aOps, FeatureNEON, FeaturePerfMon,
+                                    FeatureMTE, FeatureSSBS];
 
   // ETE and TRBE are future architecture extensions. We temporarily enable them
   // by default for users targeting generic AArch64. The extensions do not
   // affect code generated by the compiler and can be used only by explicitly
   // mentioning the new system register names in assembly.
   list<SubtargetFeature> Generic = [FeatureFPARMv8, FeatureNEON, FeaturePerfMon, FeatureETE];
 }
 
 
 def : ProcessorModel<"generic", CortexA55Model, ProcessorFeatures.Generic,
                      [FeatureFuseAES, FeaturePostRAScheduler]>;
 def : ProcessorModel<"cortex-a35", CortexA53Model, ProcessorFeatures.A53,
                      [TuneA35]>;
 def : ProcessorModel<"cortex-a34", CortexA53Model, ProcessorFeatures.A53,
                      [TuneA35]>;
 def : ProcessorModel<"cortex-a53", CortexA53Model, ProcessorFeatures.A53,
                      [TuneA53]>;
 def : ProcessorModel<"cortex-a55", CortexA55Model, ProcessorFeatures.A55,
                      [TuneA55]>;
 def : ProcessorModel<"cortex-a510", CortexA55Model, ProcessorFeatures.A510,
                      [TuneA510]>;
 def : ProcessorModel<"cortex-a57", CortexA57Model, ProcessorFeatures.A53,
                      [TuneA57]>;
 def : ProcessorModel<"cortex-a65", CortexA53Model, ProcessorFeatures.A65,
                      [TuneA65]>;
 def : ProcessorModel<"cortex-a65ae", CortexA53Model, ProcessorFeatures.A65,
                      [TuneA65]>;
 def : ProcessorModel<"cortex-a72", CortexA57Model, ProcessorFeatures.A53,
                      [TuneA72]>;
 def : ProcessorModel<"cortex-a73", CortexA57Model, ProcessorFeatures.A53,
                      [TuneA73]>;
 def : ProcessorModel<"cortex-a75", CortexA57Model, ProcessorFeatures.A55,
                      [TuneA75]>;
 def : ProcessorModel<"cortex-a76", CortexA57Model, ProcessorFeatures.A76,
                      [TuneA76]>;
 def : ProcessorModel<"cortex-a76ae", CortexA57Model, ProcessorFeatures.A76,
                      [TuneA76]>;
 def : ProcessorModel<"cortex-a77", CortexA57Model, ProcessorFeatures.A77,
                      [TuneA77]>;
 def : ProcessorModel<"cortex-a78", CortexA57Model, ProcessorFeatures.A78,
                      [TuneA78]>;
 def : ProcessorModel<"cortex-a78c", CortexA57Model, ProcessorFeatures.A78C,
                      [TuneA78C]>;
 def : ProcessorModel<"cortex-a710", CortexA57Model, ProcessorFeatures.A710,
                      [TuneA710]>;
 def : ProcessorModel<"cortex-r82", CortexA55Model, ProcessorFeatures.R82,
                      [TuneR82]>;
 def : ProcessorModel<"cortex-x1", CortexA57Model, ProcessorFeatures.X1,
                      [TuneX1]>;
 def : ProcessorModel<"cortex-x1c", CortexA57Model, ProcessorFeatures.X1C,
                      [TuneX1]>;
 def : ProcessorModel<"cortex-x2", CortexA57Model, ProcessorFeatures.X2,
                      [TuneX2]>;
 def : ProcessorModel<"neoverse-e1", CortexA53Model,
                      ProcessorFeatures.NeoverseE1, [TuneNeoverseE1]>;
 def : ProcessorModel<"neoverse-n1", CortexA57Model,
                      ProcessorFeatures.NeoverseN1, [TuneNeoverseN1]>;
 def : ProcessorModel<"neoverse-n2", CortexA57Model,
                      ProcessorFeatures.NeoverseN2, [TuneNeoverseN2]>;
 def : ProcessorModel<"neoverse-512tvb", CortexA57Model,
                      ProcessorFeatures.Neoverse512TVB, [TuneNeoverse512TVB]>;
 def : ProcessorModel<"neoverse-v1", CortexA57Model,
                      ProcessorFeatures.NeoverseV1, [TuneNeoverseV1]>;
 def : ProcessorModel<"exynos-m3", ExynosM3Model, ProcessorFeatures.ExynosM3,
                      [TuneExynosM3]>;
 def : ProcessorModel<"exynos-m4", ExynosM4Model, ProcessorFeatures.ExynosM4,
                      [TuneExynosM4]>;
 def : ProcessorModel<"exynos-m5", ExynosM5Model, ProcessorFeatures.ExynosM4,
                      [TuneExynosM4]>;
 def : ProcessorModel<"falkor", FalkorModel, ProcessorFeatures.Falkor,
                      [TuneFalkor]>;
 def : ProcessorModel<"saphira", FalkorModel, ProcessorFeatures.Saphira,
                      [TuneSaphira]>;
 def : ProcessorModel<"kryo", KryoModel, ProcessorFeatures.A53, [TuneKryo]>;
 
 // Cavium ThunderX/ThunderX T8X  Processors
 def : ProcessorModel<"thunderx", ThunderXT8XModel,  ProcessorFeatures.ThunderX,
                      [TuneThunderX]>;
 def : ProcessorModel<"thunderxt88", ThunderXT8XModel,
                      ProcessorFeatures.ThunderX, [TuneThunderXT88]>;
 def : ProcessorModel<"thunderxt81", ThunderXT8XModel,
                      ProcessorFeatures.ThunderX, [TuneThunderXT81]>;
 def : ProcessorModel<"thunderxt83", ThunderXT8XModel,
                      ProcessorFeatures.ThunderX, [TuneThunderXT83]>;
 // Cavium ThunderX2T9X  Processors. Formerly Broadcom Vulcan.
 def : ProcessorModel<"thunderx2t99", ThunderX2T99Model,
                      ProcessorFeatures.ThunderX2T99, [TuneThunderX2T99]>;
 // Marvell ThunderX3T110 Processors.
 def : ProcessorModel<"thunderx3t110", ThunderX3T110Model,
                      ProcessorFeatures.ThunderX3T110, [TuneThunderX3T110]>;
 def : ProcessorModel<"tsv110", TSV110Model, ProcessorFeatures.TSV110,
                      [TuneTSV110]>;
 
 // Support cyclone as an alias for apple-a7 so we can still LTO old bitcode.
 def : ProcessorModel<"cyclone", CycloneModel, ProcessorFeatures.AppleA7,
                      [TuneAppleA7]>;
 
 // iPhone and iPad CPUs
 def : ProcessorModel<"apple-a7", CycloneModel, ProcessorFeatures.AppleA7,
                      [TuneAppleA7]>;
 def : ProcessorModel<"apple-a8", CycloneModel, ProcessorFeatures.AppleA7,
                      [TuneAppleA7]>;
 def : ProcessorModel<"apple-a9", CycloneModel, ProcessorFeatures.AppleA7,
                      [TuneAppleA7]>;
 def : ProcessorModel<"apple-a10", CycloneModel, ProcessorFeatures.AppleA10,
                      [TuneAppleA10]>;
 def : ProcessorModel<"apple-a11", CycloneModel, ProcessorFeatures.AppleA11,
                      [TuneAppleA11]>;
 def : ProcessorModel<"apple-a12", CycloneModel, ProcessorFeatures.AppleA12,
                      [TuneAppleA12]>;
 def : ProcessorModel<"apple-a13", CycloneModel, ProcessorFeatures.AppleA13,
                      [TuneAppleA13]>;
 def : ProcessorModel<"apple-a14", CycloneModel, ProcessorFeatures.AppleA14,
                      [TuneAppleA14]>;
 
 // Mac CPUs
 def : ProcessorModel<"apple-m1", CycloneModel, ProcessorFeatures.AppleA14,
                      [TuneAppleA14]>;
 
 // watch CPUs.
 def : ProcessorModel<"apple-s4", CycloneModel, ProcessorFeatures.AppleA12,
                      [TuneAppleA12]>;
 def : ProcessorModel<"apple-s5", CycloneModel, ProcessorFeatures.AppleA12,
                      [TuneAppleA12]>;
 
 // Alias for the latest Apple processor model supported by LLVM.
 def : ProcessorModel<"apple-latest", CycloneModel, ProcessorFeatures.AppleA14,
                      [TuneAppleA14]>;
 
 // Fujitsu A64FX
 def : ProcessorModel<"a64fx", A64FXModel, ProcessorFeatures.A64FX,
                      [TuneA64FX]>;
 
 // Nvidia Carmel
 def : ProcessorModel<"carmel", NoSchedModel, ProcessorFeatures.Carmel,
                      [TuneCarmel]>;
 
+// Ampere Computing
+def : ProcessorModel<"ampere1", Ampere1Model, ProcessorFeatures.Ampere1,
+                     [TuneAmpere1]>;
+
 //===----------------------------------------------------------------------===//
 // Assembly parser
 //===----------------------------------------------------------------------===//
 
 def GenericAsmParserVariant : AsmParserVariant {
   int Variant = 0;
   string Name = "generic";
   string BreakCharacters = ".";
   string TokenizingCharacters = "[]*!/";
 }
 
 def AppleAsmParserVariant : AsmParserVariant {
   int Variant = 1;
   string Name = "apple-neon";
   string BreakCharacters = ".";
   string TokenizingCharacters = "[]*!/";
 }
 
 //===----------------------------------------------------------------------===//
 // Assembly printer
 //===----------------------------------------------------------------------===//
 // AArch64 Uses the MC printer for asm output, so make sure the TableGen
 // AsmWriter bits get associated with the correct class.
 def GenericAsmWriter : AsmWriter {
   string AsmWriterClassName  = "InstPrinter";
   int PassSubtarget = 1;
   int Variant = 0;
   bit isMCAsmWriter = 1;
 }
 
 def AppleAsmWriter : AsmWriter {
   let AsmWriterClassName = "AppleInstPrinter";
   int PassSubtarget = 1;
   int Variant = 1;
   int isMCAsmWriter = 1;
 }
 
 //===----------------------------------------------------------------------===//
 // Target Declaration
 //===----------------------------------------------------------------------===//
 
 def AArch64 : Target {
   let InstructionSet = AArch64InstrInfo;
   let AssemblyParserVariants = [GenericAsmParserVariant, AppleAsmParserVariant];
   let AssemblyWriters = [GenericAsmWriter, AppleAsmWriter];
   let AllowRegisterRenaming = 1;
 }
 
 //===----------------------------------------------------------------------===//
 // Pfm Counters
 //===----------------------------------------------------------------------===//
 
 include "AArch64PfmCounters.td"
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64SchedAmpere1.td b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64SchedAmpere1.td
new file mode 100644
index 000000000000..32f7299fbf87
--- /dev/null
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64SchedAmpere1.td
@@ -0,0 +1,1136 @@
+//=- AArch64SchedAmpere1.td - Ampere-1 scheduling def -----*- tablegen -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the machine model for the Ampere Computing Ampere-1 to
+// support instruction scheduling and other instruction cost heuristics.
+//
+//===----------------------------------------------------------------------===//
+
+// The Ampere-1 core is an out-of-order micro-architecture.  The front
+// end has branch prediction, with a 10-cycle recovery time from a
+// mispredicted branch.  Instructions coming out of the front end are
+// decoded into internal micro-ops (uops).
+
+def Ampere1Model : SchedMachineModel {
+  let IssueWidth            =   4;  // 4-way decode and dispatch
+  let MicroOpBufferSize     = 174;  // micro-op re-order buffer size
+  let LoadLatency           =   4;  // Optimistic load latency
+  let MispredictPenalty     =  10;  // Branch mispredict penalty
+  let LoopMicroOpBufferSize =  32;  // Instruction queue size
+  let CompleteModel = 1;
+
+  list<Predicate> UnsupportedFeatures = !listconcat(SVEUnsupported.F,
+                                                    SMEUnsupported.F);
+}
+
+let SchedModel = Ampere1Model in {
+
+//===----------------------------------------------------------------------===//
+// Define each kind of processor resource and number available on Ampere-1.
+// Ampere-1 has 12 pipelines that 8 independent scheduler (4 integer, 2 FP,
+// and 2 memory) issue into.  The integer and FP schedulers can each issue
+// one uop per cycle, while the memory schedulers can each issue one load
+// and one store address calculation per cycle.
+
+def Ampere1UnitA  : ProcResource<2>;  // integer single-cycle, branch, and flags r/w
+def Ampere1UnitB  : ProcResource<2>;  // integer single-cycle, and complex shifts
+def Ampere1UnitBS : ProcResource<1>;  // integer multi-cycle
+def Ampere1UnitL  : ProcResource<2>;  // load
+def Ampere1UnitS  : ProcResource<2>;  // store address calculation
+def Ampere1UnitX  : ProcResource<1>;  // FP and vector operations, and flag write
+def Ampere1UnitY  : ProcResource<1>;  // FP and vector operations, and crypto
+def Ampere1UnitZ  : ProcResource<1>;  // FP store data and FP-to-integer moves
+
+def Ampere1UnitAB : ProcResGroup<[Ampere1UnitA, Ampere1UnitB]>;
+def Ampere1UnitXY : ProcResGroup<[Ampere1UnitX, Ampere1UnitY]>;
+
+//===----------------------------------------------------------------------===//
+// Define customized scheduler read/write types specific to the Ampere-1.
+
+def Ampere1Write_1cyc_1A : SchedWriteRes<[Ampere1UnitA]> {
+  let Latency = 1;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_1cyc_2A : SchedWriteRes<[Ampere1UnitA, Ampere1UnitA]> {
+  let Latency = 1;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_1cyc_1B : SchedWriteRes<[Ampere1UnitB]> {
+  let Latency = 1;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_1cyc_1AB : SchedWriteRes<[Ampere1UnitAB]> {
+  let Latency = 1;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_1cyc_1L : SchedWriteRes<[Ampere1UnitL]> {
+  let Latency = 1;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_1cyc_1S : SchedWriteRes<[Ampere1UnitS]> {
+  let Latency = 1;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_1cyc_2S : SchedWriteRes<[Ampere1UnitS, Ampere1UnitS]> {
+  let Latency = 1;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_2cyc_1Y : SchedWriteRes<[Ampere1UnitY]> {
+  let Latency = 2;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_2cyc_2AB : SchedWriteRes<[Ampere1UnitAB, Ampere1UnitAB]> {
+  let Latency = 2;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_2cyc_1B_1AB : SchedWriteRes<[Ampere1UnitB, Ampere1UnitAB]> {
+  let Latency = 2;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_2cyc_1B_1A : SchedWriteRes<[Ampere1UnitB, Ampere1UnitA]> {
+  let Latency = 2;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_2cyc_1AB_1A : SchedWriteRes<[Ampere1UnitAB, Ampere1UnitA]> {
+  let Latency = 2;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_2cyc_1AB_1L : SchedWriteRes<[Ampere1UnitAB, Ampere1UnitL]> {
+  let Latency = 2;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_2cyc_1AB_2S : SchedWriteRes<[Ampere1UnitAB, Ampere1UnitS,
+                                                             Ampere1UnitS]> {
+  let Latency = 2;
+  let NumMicroOps = 3;
+}
+
+def Ampere1Write_2cyc_1AB_1S_1Z : SchedWriteRes<[Ampere1UnitAB, Ampere1UnitS,
+                                                                Ampere1UnitZ]> {
+  let Latency = 2;
+  let NumMicroOps = 3;
+}
+
+def Ampere1Write_2cyc_1B_1S : SchedWriteRes<[Ampere1UnitB, Ampere1UnitS]> {
+  let Latency = 2;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_2cyc_1XY : SchedWriteRes<[Ampere1UnitXY]> {
+  let Latency = 2;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_2cyc_1S_1Z : SchedWriteRes<[Ampere1UnitS, Ampere1UnitZ]> {
+  let Latency = 2;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_3cyc_1BS : SchedWriteRes<[Ampere1UnitBS]> {
+  let Latency = 3;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_3cyc_1XY : SchedWriteRes<[Ampere1UnitXY]> {
+  let Latency = 3;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_3cyc_1B_1S_1AB : SchedWriteRes<[Ampere1UnitB, Ampere1UnitS,
+                                                               Ampere1UnitAB]> {
+  let Latency = 2;
+  let NumMicroOps = 3;
+}
+
+def Ampere1Write_3cyc_1S_2Z : SchedWriteRes<[Ampere1UnitS, Ampere1UnitZ, Ampere1UnitZ]> {
+  let Latency = 2;
+  let NumMicroOps = 3;
+}
+
+def Ampere1Write_3cyc_2S_2Z : SchedWriteRes<[Ampere1UnitS, Ampere1UnitS,
+                                             Ampere1UnitZ, Ampere1UnitZ]> {
+  let Latency = 2;
+  let NumMicroOps = 4;
+}
+
+def Ampere1Write_4cyc_1BS : SchedWriteRes<[Ampere1UnitBS]> {
+  let Latency = 4;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_4cyc_1L : SchedWriteRes<[Ampere1UnitL]> {
+  let Latency = 4;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_4cyc_1X : SchedWriteRes<[Ampere1UnitX]> {
+  let Latency = 4;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_4cyc_1Y : SchedWriteRes<[Ampere1UnitY]> {
+  let Latency = 4;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_4cyc_1Z : SchedWriteRes<[Ampere1UnitZ]> {
+  let Latency = 4;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_4cyc_2L : SchedWriteRes<[Ampere1UnitL, Ampere1UnitL]> {
+  let Latency = 4;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_4cyc_1XY : SchedWriteRes<[Ampere1UnitXY]> {
+  let Latency = 4;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_4cyc_2XY : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 4;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_4cyc_1XY_1S_1Z : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitS, Ampere1UnitZ]> {
+  let Latency = 4;
+  let NumMicroOps = 3;
+}
+
+def Ampere1Write_4cyc_3S_3Z : SchedWriteRes<[Ampere1UnitS, Ampere1UnitS, Ampere1UnitS,
+                                             Ampere1UnitZ, Ampere1UnitZ, Ampere1UnitZ]> {
+  let Latency = 4;
+  let NumMicroOps = 6;
+}
+
+def Ampere1Write_5cyc_1AB_1L : SchedWriteRes<[Ampere1UnitAB, Ampere1UnitL]> {
+  let Latency = 5;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_5cyc_1BS : SchedWriteRes<[Ampere1UnitBS]> {
+  let Latency = 5;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_5cyc_1X : SchedWriteRes<[Ampere1UnitX]> {
+  let Latency = 5;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_5cyc_1L : SchedWriteRes<[Ampere1UnitL]> {
+  let Latency = 5;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_5cyc_2L : SchedWriteRes<[Ampere1UnitL, Ampere1UnitL]> {
+  let Latency = 5;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_5cyc_1L_1BS : SchedWriteRes<[Ampere1UnitL, Ampere1UnitBS]> {
+  let Latency = 5;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_5cyc_1XY : SchedWriteRes<[Ampere1UnitXY]> {
+  let Latency = 5;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_5cyc_2XY : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 5;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_5cyc_4S_4Z : SchedWriteRes<[Ampere1UnitS, Ampere1UnitS,
+                                             Ampere1UnitS, Ampere1UnitS,
+                                             Ampere1UnitZ, Ampere1UnitZ,
+                                             Ampere1UnitZ, Ampere1UnitZ]> {
+  let Latency = 5;
+  let NumMicroOps = 8;
+}
+
+def Ampere1Write_5cyc_2XY_2S_2Z : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY,
+                                                 Ampere1UnitS, Ampere1UnitS,
+                                                 Ampere1UnitZ, Ampere1UnitZ]> {
+  let Latency = 5;
+  let NumMicroOps = 6;
+}
+
+def Ampere1Write_6cyc_2XY_2S_2Z : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY,
+                                                 Ampere1UnitS, Ampere1UnitS,
+                                                 Ampere1UnitZ, Ampere1UnitZ]> {
+  let Latency = 6;
+  let NumMicroOps = 6;
+}
+
+def Ampere1Write_6cyc_3XY_3S_3Z : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY, Ampere1UnitXY,
+                                                 Ampere1UnitS, Ampere1UnitS, Ampere1UnitS,
+                                                 Ampere1UnitZ, Ampere1UnitZ, Ampere1UnitZ]> {
+  let Latency = 6;
+  let NumMicroOps = 9;
+}
+
+def Ampere1Write_6cyc_1AB_1L : SchedWriteRes<[Ampere1UnitAB, Ampere1UnitL]> {
+  let Latency = 6;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_6cyc_1XY : SchedWriteRes<[Ampere1UnitXY]> {
+  let Latency = 6;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_6cyc_2XY : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 6;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_6cyc_3XY : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 6;
+  let NumMicroOps = 3;
+}
+
+def Ampere1Write_6cyc_3L : SchedWriteRes<[Ampere1UnitL, Ampere1UnitL, Ampere1UnitL]> {
+  let Latency = 6;
+  let NumMicroOps = 3;
+}
+
+def Ampere1Write_6cyc_4L : SchedWriteRes<[Ampere1UnitL, Ampere1UnitL,
+                                          Ampere1UnitL, Ampere1UnitL]> {
+  let Latency = 6;
+  let NumMicroOps = 4;
+}
+
+def Ampere1Write_6cyc_1XY_1Z : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitZ]> {
+  let Latency = 6;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_7cyc_1BS : SchedWriteRes<[Ampere1UnitBS]> {
+  let Latency = 7;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_7cyc_1BS_1XY : SchedWriteRes<[Ampere1UnitBS, Ampere1UnitXY]> {
+  let Latency = 7;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_7cyc_1L_1XY : SchedWriteRes<[Ampere1UnitL, Ampere1UnitXY]> {
+  let Latency = 7;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_7cyc_2L_2XY : SchedWriteRes<[Ampere1UnitL, Ampere1UnitL,
+                                              Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 7;
+  let NumMicroOps = 4;
+}
+
+def Ampere1Write_7cyc_2XY : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 7;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_7cyc_4XY_4S_4Z : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY,
+                                                 Ampere1UnitXY, Ampere1UnitXY,
+                                                 Ampere1UnitS, Ampere1UnitS,
+                                                 Ampere1UnitS, Ampere1UnitS,
+                                                 Ampere1UnitZ, Ampere1UnitZ,
+                                                 Ampere1UnitZ, Ampere1UnitZ]> {
+  let Latency = 7;
+  let NumMicroOps = 12;
+}
+
+def Ampere1Write_8cyc_1BS_1A : SchedWriteRes<[Ampere1UnitBS, Ampere1UnitA]> {
+  let Latency = 8;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_8cyc_1BS_2A : SchedWriteRes<[Ampere1UnitBS, Ampere1UnitA,
+                                                             Ampere1UnitA]> {
+  let Latency = 8;
+  let NumMicroOps = 3;
+}
+
+def Ampere1Write_8cyc_2XY : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 8;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_8cyc_4XY : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY,
+                                           Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 8;
+  let NumMicroOps = 4;
+}
+
+def Ampere1Write_8cyc_3L_3XY : SchedWriteRes<[Ampere1UnitL, Ampere1UnitL, Ampere1UnitL,
+                                              Ampere1UnitXY, Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 8;
+  let NumMicroOps = 6;
+}
+
+def Ampere1Write_8cyc_4L_4XY : SchedWriteRes<[Ampere1UnitL, Ampere1UnitL,
+                                              Ampere1UnitL, Ampere1UnitL,
+                                              Ampere1UnitXY, Ampere1UnitXY,
+                                              Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 8;
+  let NumMicroOps = 8;
+}
+
+def Ampere1Write_9cyc_3L_3XY : SchedWriteRes<[Ampere1UnitL, Ampere1UnitL, Ampere1UnitL,
+                                              Ampere1UnitXY, Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 9;
+  let NumMicroOps = 6;
+}
+
+def Ampere1Write_9cyc_4L_4XY : SchedWriteRes<[Ampere1UnitL, Ampere1UnitL,
+                                              Ampere1UnitL, Ampere1UnitL,
+                                              Ampere1UnitXY, Ampere1UnitXY,
+                                              Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 9;
+  let NumMicroOps = 8;
+}
+
+def Ampere1Write_9cyc_3XY : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 9;
+  let NumMicroOps = 3;
+}
+
+def Ampere1Write_9cyc_2L_3XY : SchedWriteRes<[Ampere1UnitL, Ampere1UnitL,
+                                              Ampere1UnitXY, Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 9;
+  let NumMicroOps = 5;
+}
+
+def Ampere1Write_9cyc_6XY_4S_4Z : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY,
+                                                 Ampere1UnitXY, Ampere1UnitXY,
+                                                 Ampere1UnitXY, Ampere1UnitXY,
+                                                 Ampere1UnitS, Ampere1UnitS,
+                                                 Ampere1UnitS, Ampere1UnitS,
+                                                 Ampere1UnitZ, Ampere1UnitZ,
+                                                 Ampere1UnitZ, Ampere1UnitZ]> {
+  let Latency = 9;
+  let NumMicroOps = 14;
+}
+
+def Ampere1Write_9cyc_8XY_4S_4Z : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY,
+                                                 Ampere1UnitXY, Ampere1UnitXY,
+                                                 Ampere1UnitXY, Ampere1UnitXY,
+                                                 Ampere1UnitXY, Ampere1UnitXY,
+                                                 Ampere1UnitS, Ampere1UnitS,
+                                                 Ampere1UnitS, Ampere1UnitS,
+                                                 Ampere1UnitZ, Ampere1UnitZ,
+                                                 Ampere1UnitZ, Ampere1UnitZ]> {
+  let Latency = 9;
+  let NumMicroOps = 16;
+}
+
+def Ampere1Write_10cyc_2XY : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 10;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_10cyc_1XY_1Z : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitZ]> {
+  let Latency = 10;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_10cyc_1X_1Z : SchedWriteRes<[Ampere1UnitX, Ampere1UnitZ]> {
+  let Latency = 10;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_10cyc_3L_3XY : SchedWriteRes<[Ampere1UnitL, Ampere1UnitL, Ampere1UnitL,
+                                               Ampere1UnitXY, Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 10;
+  let NumMicroOps = 6;
+}
+
+def Ampere1Write_10cyc_1A_1BS_1X : SchedWriteRes<[Ampere1UnitA, Ampere1UnitBS, Ampere1UnitX]> {
+  let Latency = 10;
+  let NumMicroOps = 3;
+}
+
+def Ampere1Write_10cyc_1A_1BS_1XY : SchedWriteRes<[Ampere1UnitA, Ampere1UnitBS, Ampere1UnitXY]> {
+  let Latency = 10;
+  let NumMicroOps = 3;
+}
+
+def Ampere1Write_11cyc_1BS_1L : SchedWriteRes<[Ampere1UnitBS, Ampere1UnitL]> {
+  let Latency = 11;
+  let NumMicroOps = 2;
+}
+
+def Ampere1Write_11cyc_1A_1BS_1X : SchedWriteRes<[Ampere1UnitA, Ampere1UnitBS, Ampere1UnitX]> {
+  let Latency = 11;
+  let NumMicroOps = 3;
+}
+
+def Ampere1Write_11cyc_1A_1BS_1XY : SchedWriteRes<[Ampere1UnitA, Ampere1UnitBS, Ampere1UnitXY]> {
+  let Latency = 11;
+  let NumMicroOps = 3;
+}
+
+def Ampere1Write_11cyc_4L_8XY : SchedWriteRes<[Ampere1UnitL, Ampere1UnitL,
+                                               Ampere1UnitL, Ampere1UnitL,
+                                               Ampere1UnitXY, Ampere1UnitXY,
+                                               Ampere1UnitXY, Ampere1UnitXY,
+                                               Ampere1UnitXY, Ampere1UnitXY,
+                                               Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 11;
+  let NumMicroOps = 12;
+}
+
+def Ampere1Write_12cyc_4L_8XY : SchedWriteRes<[Ampere1UnitL, Ampere1UnitL,
+                                               Ampere1UnitL, Ampere1UnitL,
+                                               Ampere1UnitXY, Ampere1UnitXY,
+                                               Ampere1UnitXY, Ampere1UnitXY,
+                                               Ampere1UnitXY, Ampere1UnitXY,
+                                               Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 12;
+  let NumMicroOps = 12;
+}
+
+def Ampere1Write_12cyc_3XY : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 12;
+  let NumMicroOps = 3;
+}
+
+def Ampere1Write_12cyc_4XY : SchedWriteRes<[Ampere1UnitXY, Ampere1UnitXY,
+                                            Ampere1UnitXY, Ampere1UnitXY]> {
+  let Latency = 12;
+  let NumMicroOps = 4;
+}
+
+def Ampere1Write_18cyc_1BS : SchedWriteRes<[Ampere1UnitBS]> {
+  let Latency = 18;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_19cyc_1XY : SchedWriteRes<[Ampere1UnitXY]> {
+  let Latency = 19;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_25cyc_1XY : SchedWriteRes<[Ampere1UnitXY]> {
+  let Latency = 25;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_32cyc_1XY : SchedWriteRes<[Ampere1UnitXY]> {
+  let Latency = 32;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_34cyc_1BS : SchedWriteRes<[Ampere1UnitBS]> {
+  let Latency = 34;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_34cyc_1XY : SchedWriteRes<[Ampere1UnitXY]> {
+  let Latency = 34;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_39cyc_1XY : SchedWriteRes<[Ampere1UnitXY]> {
+  let Latency = 39;
+  let NumMicroOps = 1;
+}
+
+def Ampere1Write_62cyc_1XY : SchedWriteRes<[Ampere1UnitXY]> {
+  let Latency = 62;
+  let NumMicroOps = 1;
+}
+
+// For basic arithmetic, we have more flexibility for short shifts (LSL shift <= 4),
+// which are a single uop, and for extended registers, which have full flexibility
+// across Unit A or B for both uops.
+def Ampere1Write_Arith : SchedWriteVariant<[
+                                SchedVar<RegExtendedPred, [Ampere1Write_2cyc_2AB]>,
+                                SchedVar<AmpereCheapLSL,  [Ampere1Write_1cyc_1AB]>,
+                                SchedVar<NoSchedPred,     [Ampere1Write_2cyc_1B_1AB]>]>;
+
+def Ampere1Write_ArithFlagsetting : SchedWriteVariant<[
+                                SchedVar<RegExtendedPred, [Ampere1Write_2cyc_1AB_1A]>,
+                                SchedVar<AmpereCheapLSL,  [Ampere1Write_1cyc_1A]>,
+                                SchedVar<NoSchedPred,     [Ampere1Write_2cyc_1B_1A]>]>;
+
+//===----------------------------------------------------------------------===//
+// Map the target-defined scheduler read/write resources and latencies for Ampere-1.
+// This provides a coarse model, which is then specialised below.
+
+def : WriteRes<WriteImm,   [Ampere1UnitAB]>;  // MOVN, MOVZ
+def : WriteRes<WriteI,     [Ampere1UnitAB]>;  // ALU
+def : WriteRes<WriteISReg, [Ampere1UnitB, Ampere1UnitA]> {
+  let Latency = 2;
+  let NumMicroOps = 2;
+}  // ALU of Shifted-Reg
+def : WriteRes<WriteIEReg, [Ampere1UnitAB, Ampere1UnitA]> {
+  let Latency = 2;
+  let NumMicroOps = 2;
+}  // ALU of Extended-Reg
+def : WriteRes<WriteExtr,  [Ampere1UnitB]>;  // EXTR shifts a reg pair
+def : WriteRes<WriteIS,    [Ampere1UnitB]>;  // Shift/Scale
+def : WriteRes<WriteID32,  [Ampere1UnitBS]> {
+  let Latency = 18;
+}  // 32-bit Divide
+def : WriteRes<WriteID64,  [Ampere1UnitBS]> {
+  let Latency = 34;
+}  // 64-bit Divide
+def : WriteRes<WriteIM32,  [Ampere1UnitBS]> {
+  let Latency = 3;
+}  // 32-bit Multiply
+def : WriteRes<WriteIM64,  [Ampere1UnitBS]> {
+  let Latency = 3;
+}  // 32-bit Multiply
+def : WriteRes<WriteBr,    [Ampere1UnitA]>;
+def : WriteRes<WriteBrReg, [Ampere1UnitA, Ampere1UnitA]>;
+def : WriteRes<WriteLD,    [Ampere1UnitL]> {
+  let Latency = 4;
+}  // Load from base addr plus immediate offset
+def : WriteRes<WriteST,    [Ampere1UnitS]> {
+  let Latency = 1;
+}  // Store to base addr plus immediate offset
+def : WriteRes<WriteSTP,   [Ampere1UnitS, Ampere1UnitS]> {
+  let Latency = 1;
+  let NumMicroOps = 2;
+}  // Store a register pair.
+def : WriteRes<WriteAdr,   [Ampere1UnitAB]>;
+def : WriteRes<WriteLDIdx, [Ampere1UnitAB, Ampere1UnitS]> {
+  let Latency = 5;
+  let NumMicroOps = 2;
+}  // Load from a register index (maybe scaled).
+def : WriteRes<WriteSTIdx, [Ampere1UnitS, Ampere1UnitS]> {
+  let Latency = 1;
+  let NumMicroOps = 2;
+}  // Store to a register index (maybe scaled).
+def : WriteRes<WriteF,  [Ampere1UnitXY]> {
+  let Latency = 2;
+}  // General floating-point ops.
+def : WriteRes<WriteFCmp,  [Ampere1UnitX]> {
+  let Latency = 5;
+}  // Floating-point compare.
+def : WriteRes<WriteFCvt,  [Ampere1UnitXY]> {
+  let Latency = 6;
+}  // Float conversion.
+def : WriteRes<WriteFCopy, [Ampere1UnitXY]> {
+}  // Float-int register copy.
+def : WriteRes<WriteFImm,  [Ampere1UnitXY]> {
+  let Latency = 2;
+}  // Float-int register copy.
+def : WriteRes<WriteFMul,  [Ampere1UnitXY]> {
+  let Latency = 5;
+}  // Floating-point multiply.
+def : WriteRes<WriteFDiv,  [Ampere1UnitXY]> {
+  let Latency = 34;
+}  // Floating-point division.
+def : WriteRes<WriteVd,    [Ampere1UnitXY]> {
+  let Latency = 3;
+}  // 64bit Vector D ops.
+def : WriteRes<WriteVq,    [Ampere1UnitXY]> {
+  let Latency = 3;
+}  // 128bit Vector Q ops.
+def : WriteRes<WriteVLD,   [Ampere1UnitL, Ampere1UnitL]> {
+  let Latency = 5;
+}  // Vector loads.
+def : WriteRes<WriteVST,   [Ampere1UnitS, Ampere1UnitZ]> {
+  let Latency = 2;
+}  // Vector stores.
+
+def : WriteRes<WriteAtomic,  []> { let Unsupported = 1; }
+
+def : WriteRes<WriteSys,     []> { let Latency = 1; }
+def : WriteRes<WriteBarrier, []> { let Latency = 1; }
+def : WriteRes<WriteHint,    []> { let Latency = 1; }
+
+def : WriteRes<WriteLDHi,    []> {
+  let Latency = 4;
+}  // The second register of a load-pair: LDP,LDPSW,LDNP,LDXP,LDAXP
+
+// Forwarding logic.
+def : ReadAdvance<ReadI,       0>;
+def : ReadAdvance<ReadISReg,   0>;
+def : ReadAdvance<ReadIEReg,   0>;
+def : ReadAdvance<ReadIM,      0>;
+def : ReadAdvance<ReadIMA,     1, [WriteIM32, WriteIM64]>;
+def : ReadAdvance<ReadID,      0>;
+def : ReadAdvance<ReadExtrHi,  0>;
+def : ReadAdvance<ReadST,      0>;
+def : ReadAdvance<ReadAdrBase, 0>;
+def : ReadAdvance<ReadVLD,     0>;
+
+//===----------------------------------------------------------------------===//
+// Specialising the scheduling model further for Ampere-1.
+
+def : InstRW<[Ampere1Write_1cyc_1AB], (instrs COPY)>;
+
+// Branch instructions
+def : InstRW<[Ampere1Write_1cyc_1A], (instrs Bcc, BL, RET)>;
+def : InstRW<[Ampere1Write_1cyc_1A],
+        (instrs CBZW, CBZX, CBNZW, CBNZX, TBZW, TBZX, TBNZW, TBNZX)>;
+def : InstRW<[Ampere1Write_1cyc_2A], (instrs BLR)>;
+
+// Cryptography instructions
+// -- AES encryption/decryption
+def : InstRW<[Ampere1Write_2cyc_1XY], (instregex "^AES[DE]")>;
+def : InstRW<[Ampere1Write_2cyc_1XY], (instregex "^AESI?MC")>;
+// -- Polynomial multiplication
+def : InstRW<[Ampere1Write_2cyc_1XY], (instregex "^PMUL", "^PMULL")>;
+// -- SHA-256 hash
+def : InstRW<[Ampere1Write_4cyc_1X], (instregex "^SHA256(H|H2)")>;
+// -- SHA-256 schedule update
+def : InstRW<[Ampere1Write_4cyc_1Y], (instregex "^SHA256SU[01]")>;
+// -- SHA-3 instructions
+def : InstRW<[Ampere1Write_2cyc_1XY],
+        (instregex "^BCAX", "^EOR3", "^RAX1", "^XAR")>;
+// -- SHA-512 hash
+def : InstRW<[Ampere1Write_4cyc_1X], (instregex "^SHA512(H|H2)")>;
+// -- SHA-512 schedule update
+def : InstRW<[Ampere1Write_4cyc_1Y], (instregex "^SHA512SU[01]")>;
+// -- SHA1 choose/majority/parity
+def : InstRW<[Ampere1Write_4cyc_1X], (instregex "^SHA1[CMP]")>;
+// -- SHA1 hash/schedule update
+def : InstRW<[Ampere1Write_2cyc_1Y], (instregex "^SHA1SU[01]")>;
+def : InstRW<[Ampere1Write_2cyc_1Y], (instregex "^SHA1H")>;
+
+// FP and vector load instructions
+// -- Load 1-element structure to one/all lanes
+// ---- all lanes
+def : InstRW<[Ampere1Write_7cyc_1L_1XY],
+        (instregex "^LD1Rv(8b|4h|2s|16b|8h|4s|2d)")>;
+// ---- one lane
+def : InstRW<[Ampere1Write_7cyc_1L_1XY],
+        (instregex "^LD1i(8|16|32|64)")>;
+// -- Load 1-element structure to one/all lanes, 1D size
+def : InstRW<[Ampere1Write_5cyc_1L],
+        (instregex "^LD1Rv1d")>;
+// -- Load 1-element structures to 1 register
+def : InstRW<[Ampere1Write_5cyc_1L],
+        (instregex "^LD1Onev(8b|4h|2s|1d|16b|8h|4s|2d)")>;
+// -- Load 1-element structures to 2 registers
+def : InstRW<[Ampere1Write_5cyc_2L],
+        (instregex "^LD1Twov(8b|4h|2s|1d|16b|8h|4s|2d)")>;
+// -- Load 1-element structures to 3 registers
+def : InstRW<[Ampere1Write_6cyc_3L],
+        (instregex "^LD1Threev(8b|4h|2s|1d|16b|8h|4s|2d)")>;
+// -- Load 1-element structures to 4 registers
+def : InstRW<[Ampere1Write_6cyc_4L],
+        (instregex "^LD1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)")>;
+// -- Load 2-element structure to all lanes of 2 registers, 1D size
+def : InstRW<[Ampere1Write_5cyc_2L],
+        (instregex "^LD2Rv1d")>;
+// -- Load 2-element structure to all lanes of 2 registers, other sizes
+def : InstRW<[Ampere1Write_7cyc_2L_2XY],
+        (instregex "^LD2Rv(8b|4h|2s|16b|8h|4s|2d)")>;
+// -- Load 2-element structure to one lane of 2 registers
+def : InstRW<[Ampere1Write_7cyc_2L_2XY],
+        (instregex "^LD2i(8|16|32|64)")>;
+// -- Load 2-element structures to 2 registers, 16B/8H/4S/2D size
+def : InstRW<[Ampere1Write_7cyc_2L_2XY],
+        (instregex "^LD2Twov(16b|8h|4s|2d)")>;
+// -- Load 2-element structures to 2 registers, 8B/4H/2S size
+def : InstRW<[Ampere1Write_9cyc_2L_3XY],
+        (instregex "^LD2Twov(8b|4h|2s)")>;
+// -- Load 3-element structure to all lanes of 3 registers, 1D size
+def : InstRW<[Ampere1Write_6cyc_3L],
+        (instregex "^LD3Rv1d")>;
+// -- Load 3-element structure to all lanes of 3 registers, other sizes
+def : InstRW<[Ampere1Write_8cyc_3L_3XY],
+        (instregex "^LD3Rv(8b|4h|2s|16b|8h|4s|2d)")>;
+// -- Load 3-element structure to one lane of 3 registers
+def : InstRW<[Ampere1Write_8cyc_3L_3XY],
+        (instregex "^LD3i(8|16|32|64)")>;
+// -- Load 3-element structures to 3 registers, 16B/8H/4S sizes
+def : InstRW<[Ampere1Write_9cyc_3L_3XY],
+        (instregex "^LD3Threev(16b|8h|4s)")>;
+// -- Load 3-element structures to 3 registers, 2D size
+def : InstRW<[Ampere1Write_8cyc_3L_3XY],
+        (instregex "^LD3Threev2d")>;
+// -- Load 3-element structures to 3 registers, 8B/4H/2S sizes
+def : InstRW<[Ampere1Write_10cyc_3L_3XY],
+        (instregex "^LD3Threev(8b|4h|2s)")>;
+// -- Load 4-element structure to all lanes of 4 registers, 1D size
+def : InstRW<[Ampere1Write_6cyc_4L],
+        (instregex "^LD4Rv1d")>;
+// -- Load 4-element structure to all lanes of 4 registers, other sizes
+def : InstRW<[Ampere1Write_8cyc_4L_4XY],
+        (instregex "^LD4Rv(8b|4h|2s|16b|8h|4s|2d)")>;
+// -- Load 4-element structure to one lane of 4 registers
+def : InstRW<[Ampere1Write_6cyc_4L],
+        (instregex "^LD4i(8|16|32|64)")>;
+// -- Load 4-element structures to 4 registers, 2D size
+def : InstRW<[Ampere1Write_9cyc_4L_4XY],
+        (instregex "^LD4Fourv2d")>;
+// -- Load 4-element structures to 4 registers, 2S size
+def : InstRW<[Ampere1Write_12cyc_4L_8XY],
+        (instregex "^LD4Fourv2s")>;
+// -- Load 4-element structures to 4 registers, other sizes
+def : InstRW<[Ampere1Write_11cyc_4L_8XY],
+        (instregex "^LD4Fourv(8b|4h|16b|8h|4s)")>;
+// -- Load pair, Q-form
+def : InstRW<[Ampere1Write_5cyc_2L], (instregex "LDN?PQ")>;
+// -- Load pair, S/D-form
+def : InstRW<[Ampere1Write_5cyc_1L_1BS], (instregex "LDN?P(S|D)")>;
+// -- Load register
+def : InstRW<[Ampere1Write_5cyc_1L], (instregex "LDU?R[BHSDQ]i")>;
+// -- Load register, sign-extended register
+def : InstRW<[Ampere1Write_6cyc_1AB_1L], (instregex "LDR[BHSDQ]ro(W|X)")>;
+
+// FP and vector store instructions
+// -- Store 1-element structure from one lane of 1 register
+def : InstRW<[Ampere1Write_4cyc_1XY_1S_1Z],
+        (instregex "^ST1i(8|16|32|64)")>;
+// -- Store 1-element structures from 1 register
+def : InstRW<[Ampere1Write_2cyc_1S_1Z],
+        (instregex "^ST1Onev(8b|4h|2s|1d|16b|8h|4s|2d)")>;
+// -- Store 1-element structures from 2 registers
+def : InstRW<[Ampere1Write_3cyc_2S_2Z],
+        (instregex "^ST1Twov(8b|4h|2s|1d|16b|8h|4s|2d)")>;
+// -- Store 1-element structures from 3 registers
+def : InstRW<[Ampere1Write_4cyc_3S_3Z],
+        (instregex "^ST1Threev(8b|4h|2s|1d|16b|8h|4s|2d)")>;
+// -- Store 1-element structures from 4 registers
+def : InstRW<[Ampere1Write_5cyc_4S_4Z],
+        (instregex "^ST1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)")>;
+// -- Store 2-element structure from one lane of 2 registers
+def : InstRW<[Ampere1Write_5cyc_2XY_2S_2Z],
+        (instregex "^ST2i(8|16|32|64)")>;
+// -- Store 2-element structures from 2 registers, 16B/8H/4S/2D sizes
+def : InstRW<[Ampere1Write_5cyc_2XY_2S_2Z],
+        (instregex "^ST2Twov(16b|8h|4s|2d)")>;
+// -- Store 2-element structures from 2 registers, 8B/4H/2S sizes
+def : InstRW<[Ampere1Write_6cyc_2XY_2S_2Z],
+        (instregex "^ST2Twov(8b|4h|2s)")>;
+// -- Store 3-element structure from one lane of 3 registers
+def : InstRW<[Ampere1Write_6cyc_3XY_3S_3Z],
+        (instregex "^ST3i(8|16|32|64)")>;
+// -- Store 3-element structures from 3 registers
+def : InstRW<[Ampere1Write_6cyc_3XY_3S_3Z],
+        (instregex "^ST3Threev(8b|4h|2s|1d|16b|8h|4s|2d)")>;
+// -- Store 4-element structure from one lane of 4 registers
+def : InstRW<[Ampere1Write_7cyc_4XY_4S_4Z],
+        (instregex "^ST4i(8|16|32|64)")>;
+// -- Store 4-element structures from 4 registers, 16B/8H/4S sizes
+def : InstRW<[Ampere1Write_9cyc_8XY_4S_4Z],
+        (instregex "^ST4Fourv(16b|8h|4s)")>;
+// -- Store 4-element structures from 4 registers, 2D sizes
+def : InstRW<[Ampere1Write_7cyc_4XY_4S_4Z],
+        (instregex "^ST4Fourv2d")>;
+// -- Store 4-element structures from 4 registers, 8B/4H/2S sizes
+def : InstRW<[Ampere1Write_9cyc_6XY_4S_4Z],
+        (instregex "^ST4Fourv(8b|4h|2s)")>;
+// -- Store pair, Q-form
+def : InstRW<[Ampere1Write_3cyc_2S_2Z], (instregex "^STN?PQ")>;
+// -- Store pair, S/D-form
+def : InstRW<[Ampere1Write_3cyc_1S_2Z],	(instregex "^STN?P[SD]")>;
+// -- Store register
+def : InstRW<[Ampere1Write_2cyc_1S_1Z],	(instregex "^STU?R[BHSDQ](ui|i)")>;
+// -- Store register, sign-extended register offset
+def : InstRW<[Ampere1Write_2cyc_1AB_1S_1Z], (instregex "^STR[BHSDQ]ro[XW]")>;
+
+// FP data processing, bfloat16 format
+def : InstRW<[Ampere1Write_5cyc_1XY], (instrs BFCVT)>;
+def : InstRW<[Ampere1Write_7cyc_2XY], (instrs BFCVTN, BFCVTN2)>;
+def : InstRW<[Ampere1Write_2cyc_1XY], (instregex "^BFDOTv", "^BF16DOT")>;
+def : InstRW<[Ampere1Write_4cyc_2XY], (instrs BFMMLA)>;
+def : InstRW<[Ampere1Write_5cyc_1XY], (instregex "^BFMLAL")>;
+
+// FP data processing, scalar/vector, half precision
+def : InstRW<[Ampere1Write_4cyc_1XY], (instregex "^F(ABD|ABS)v.[fi]16")>;
+def : InstRW<[Ampere1Write_4cyc_1XY],
+        (instregex "^F(ADD|ADDP|CADD|NEG|NMUL|SUB)v.[fi]16")>;
+def : InstRW<[Ampere1Write_4cyc_1XY],
+        (instregex "^F(AC|CM)(EQ|GE|GT|LE|LT)v.[fi]16")>;
+def : InstRW<[Ampere1Write_4cyc_1XY],
+        (instregex "^F(AC|CM)(EQ|GE|GT|LE|LT)16")>;
+def : InstRW<[Ampere1Write_4cyc_1X],
+        (instregex "^FCMPE?H")>;
+def : InstRW<[Ampere1Write_10cyc_1A_1BS_1X],
+        (instregex "^FCCMPE?H")>;
+def : InstRW<[Ampere1Write_10cyc_1A_1BS_1XY],
+        (instregex "^FCSELH")>;
+def : InstRW<[Ampere1Write_4cyc_1XY], (instregex "^FCVT[AMNPZ][SU]v.[if]16")>;
+def : InstRW<[Ampere1Write_4cyc_1XY], (instregex "^[SU]CVTFv.[fi]16")>;
+def : InstRW<[Ampere1Write_25cyc_1XY], (instregex "^FDIVv.[if]16", "FDIVH")>;
+def : InstRW<[Ampere1Write_4cyc_1XY], (instregex "^F(MAX|MIN)(NM)?P?v.[if]16")>;
+def : InstRW<[Ampere1Write_8cyc_2XY], (instregex "^F(MAX|MIN)(NM)?Vv4[if]16")>;
+def : InstRW<[Ampere1Write_12cyc_3XY], (instregex "^F(MAX|MIN)(NM)?Vv8[if]16")>;
+def : InstRW<[Ampere1Write_4cyc_1XY], (instregex "^FMULX?v.[if]16")>;
+def : InstRW<[Ampere1Write_4cyc_1XY], (instrs FMULX16)>;
+def : InstRW<[Ampere1Write_4cyc_1XY], (instregex "^FN?M(ADD|SUB)[H]rrr")>;
+def : InstRW<[Ampere1Write_4cyc_1XY], (instregex "^FML[AS]v.[if]16")>;
+def : InstRW<[Ampere1Write_4cyc_1XY], (instregex "^FRECPXv.[if]16")>;
+def : InstRW<[Ampere1Write_4cyc_1XY], (instregex "^F(RECP|RSQRT)S16")>;
+def : InstRW<[Ampere1Write_4cyc_1XY], (instregex "^FRINT[AIMNPXZ]v.[if]16")>;
+def : InstRW<[Ampere1Write_39cyc_1XY], (instregex "^FSQRTv.f16", "^FSQRTHr")>;
+
+// FP data processing, scalar/vector, single/double precision
+def : InstRW<[Ampere1Write_5cyc_1XY], (instregex "^F(ABD|ABS)v.[fi](32|64)")>;
+def : InstRW<[Ampere1Write_5cyc_1XY],
+        (instregex "^F(ADD|ADDP|CADD|NEG|NMUL|SUB)v.[fi](32|64)")>;
+def : InstRW<[Ampere1Write_5cyc_1XY],
+        (instregex "^F(AC|CM)(EQ|GE|GT|LE|LT)v.[fi](32|64)")>;
+def : InstRW<[Ampere1Write_5cyc_1XY],
+        (instregex "^F(AC|CM)(EQ|GE|GT|LE|LT)(32|64)")>;
+def : InstRW<[Ampere1Write_5cyc_1X],
+        (instregex "^FCMPE?(S|D)")>;
+def : InstRW<[Ampere1Write_11cyc_1A_1BS_1X],
+        (instregex "^FCCMPE?(S|D)")>;
+def : InstRW<[Ampere1Write_11cyc_1A_1BS_1XY],
+        (instregex "^FCSEL(S|D)")>;
+def : InstRW<[Ampere1Write_6cyc_1XY], (instregex "^FCVT[AMNPZ][SU]v.[if](32|64)")>;
+def : InstRW<[Ampere1Write_6cyc_1XY], (instregex "^[SU]CVTFv.[fi](32|64)")>;
+def : InstRW<[Ampere1Write_34cyc_1XY], (instregex "^FDIVv.[if](64)", "FDIVD")>;
+def : InstRW<[Ampere1Write_19cyc_1XY], (instregex "^FDIVv.[if](32)", "FDIVS")>;
+def : InstRW<[Ampere1Write_5cyc_1XY], (instregex "^F(MAX|MIN)(NM)?P?v.[if](32|64)")>;
+def : InstRW<[Ampere1Write_10cyc_2XY], (instregex "^F(MAX|MIN)(NM)?Vv.[if](32|64)")>;
+def : InstRW<[Ampere1Write_6cyc_1XY], (instregex "^FMULX?v.[if](32|64)")>;
+def : InstRW<[Ampere1Write_6cyc_1XY], (instrs FMULX32, FMULX64)>;
+def : InstRW<[Ampere1Write_5cyc_1XY], (instregex "^FN?M(ADD|SUB)[SD]rrr")>;
+def : InstRW<[Ampere1Write_5cyc_1XY], (instregex "^FML[AS]v.[if](32|64)")>;
+def : InstRW<[Ampere1Write_5cyc_1XY], (instregex "^FRECPXv.[if](32|64)")>;
+def : InstRW<[Ampere1Write_6cyc_1XY], (instregex "^F(RECP|RSQRT)S(32|64)")>;
+def : InstRW<[Ampere1Write_6cyc_1XY], (instregex "^FRINT[AIMNPXZ]v.[if](32|64)")>;
+def : InstRW<[Ampere1Write_6cyc_1XY], (instregex "^FRINT(32|64)")>;
+def : InstRW<[Ampere1Write_62cyc_1XY], (instregex "^FSQRTv.f64", "^FSQRTDr")>;
+def : InstRW<[Ampere1Write_32cyc_1XY], (instregex "^FSQRTv.f32", "^FSQRTSr")>;
+
+// FP miscellaneous instructions
+def : InstRW<[Ampere1Write_10cyc_1XY_1Z], (instregex "^FCVT[AMNPZ][SU][SU][XW][HSD]r")>;
+def : InstRW<[Ampere1Write_5cyc_1XY], (instregex "^FCVT[HSD]Hr")>;
+def : InstRW<[Ampere1Write_6cyc_1XY], (instregex "^FCVT[HSD][SD]r")>;
+def : InstRW<[Ampere1Write_6cyc_1XY], (instregex "^FCVTLv")>;
+def : InstRW<[Ampere1Write_8cyc_2XY], (instregex "^FCVT(N|XN)v")>;
+def : InstRW<[Ampere1Write_10cyc_1X_1Z], (instrs FJCVTZS)>;
+def : InstRW<[Ampere1Write_5cyc_1BS], (instregex "^FMOV[HSD][WX]r")>;
+def : InstRW<[Ampere1Write_7cyc_1BS_1XY], (instregex "^FMOVDXHighr")>;
+def : InstRW<[Ampere1Write_2cyc_1XY], (instregex "^FMOV[HSD][ri]")>;
+def : InstRW<[Ampere1Write_6cyc_1XY_1Z], (instregex "^FMOVXDHighr")>;
+def : InstRW<[Ampere1Write_4cyc_1Z], (instregex "^FMOV[WX][HSD]r")>;
+
+// Integer arithmetic and logical instructions
+def : InstRW<[Ampere1Write_1cyc_1A],
+        (instregex "ADC(W|X)r", "SBC(W|X)r")>;
+def : InstRW<[Ampere1Write_Arith],
+        (instregex "(ADD|AND|BIC|EON|EOR|ORN|ORR|SUB)(W|X)r")>;
+def : InstRW<[Ampere1Write_ArithFlagsetting],
+        (instregex "(ADD|AND|BIC|SUB)S(W|X)r")>;
+def : InstRW<[Ampere1Write_1cyc_1A],
+        (instregex "(ADC|SBC)S(W|X)r")>;
+def : InstRW<[Ampere1Write_1cyc_1A], (instrs RMIF)>;
+def : InstRW<[Ampere1Write_1cyc_1A],
+        (instregex "(CCMN|CCMP)(X|W)")>;
+def : InstRW<[Ampere1Write_1cyc_1A],
+        (instregex "(CSEL|CSINC|CSINV|CSNEG)(X|W)")>;
+def : InstRW<[Ampere1Write_18cyc_1BS], (instrs SDIVWr, UDIVWr)>;
+def : InstRW<[Ampere1Write_34cyc_1BS], (instrs SDIVXr, UDIVXr)>;
+def : InstRW<[Ampere1Write_3cyc_1BS],
+        (instregex "(S|U)MULHr")>;
+def : InstRW<[Ampere1Write_4cyc_1BS],
+        (instregex "(S|U)?M(ADD|SUB)L?r")>;
+
+// Integer load instructions
+def : InstRW<[Ampere1Write_4cyc_2L],
+        (instregex "(LDNP|LDP|LDPSW)(X|W)")>;
+def : InstRW<[Ampere1Write_4cyc_1L],
+        (instregex "LDR(B|D|H|Q|S)ui")>;
+def : InstRW<[Ampere1Write_4cyc_1L],
+        (instregex "LDR(D|Q|W|X)l")>;
+def : InstRW<[Ampere1Write_4cyc_1L],
+        (instregex "LDTR(B|H|W|X)i")>;
+def : InstRW<[Ampere1Write_4cyc_1L],
+        (instregex "LDTRS(BW|BX|HW|HX|W)i")>;
+def : InstRW<[Ampere1Write_4cyc_1L],
+        (instregex "LDUR(BB|HH|X|W)i")>;
+def : InstRW<[Ampere1Write_4cyc_1L],
+        (instregex "LDURS(BW|BX|HW|HX|W)i")>;
+def : InstRW<[Ampere1Write_5cyc_1AB_1L],
+        (instregex "LDR(HH|SHW|SHX|W|X)ro(W|X)")>;
+def : InstRW<[Ampere1Write_1cyc_1L],
+        (instrs PRFMl, PRFUMi, PRFUMi)>;
+def : InstRW<[Ampere1Write_2cyc_1AB_1L],
+        (instrs PRFMroW, PRFMroX)>;
+
+// Integer miscellaneous instructions
+def : InstRW<[Ampere1Write_1cyc_1A],  (instrs ADR, ADRP)>;
+def : InstRW<[Ampere1Write_1cyc_1B],  (instregex "EXTR(W|X)")>;
+def : InstRW<[Ampere1Write_1cyc_1B],  (instregex "(S|U)?BFM(W|X)")>;
+def : InstRW<[Ampere1Write_3cyc_1BS], (instregex "^CRC32C?[BHWX]")>;
+def : InstRW<[Ampere1Write_1cyc_1B],  (instregex "CLS(W|X)")>;
+def : InstRW<[Ampere1Write_1cyc_1A],  (instrs SETF8, SETF16)>;
+def : InstRW<[Ampere1Write_1cyc_1AB],
+        (instrs MOVKWi, MOVKXi, MOVNWi, MOVNXi, MOVZWi, MOVZXi)>;
+def : InstRW<[Ampere1Write_1cyc_1B],
+        (instregex "(RBIT|REV|REV16)(W|X)r", "REV32Xr")>;
+def : InstRW<[Ampere1Write_1cyc_1B],
+        (instregex "(ASR|LSL|LSR|ROR)V(W|X)r")>;
+
+// Integer store instructions
+def : InstRW<[Ampere1Write_1cyc_2S],  (instregex "STNP(X|W)i")>;
+def : InstRW<[Ampere1Write_2cyc_1B_1S],
+        (instrs STPWi, STPXi)>;
+def : InstRW<[Ampere1Write_3cyc_1B_1S_1AB],
+        (instregex "STP(W|X)(pre|post)")>;
+def : InstRW<[Ampere1Write_1cyc_1S],
+        (instrs STTRBi, STTRHi, STTRWi, STTRXi)>;
+def : InstRW<[Ampere1Write_1cyc_1S],
+        (instregex "STUR(BB|HH|X|W)i",
+                   "STR(X|W)ui",
+                   "STUR(BB|HH|X|W)i")>;
+def : InstRW<[Ampere1Write_1cyc_2S], (instrs STRWroX, STRXroX)>;
+def : InstRW<[Ampere1Write_2cyc_1AB_2S], (instrs STRWroW, STRXroW)>;
+
+// Pointer authentication
+//def : InstRW<[Ampere1Write_7cyc_1BS],
+//	(instrs AUTIAZ, AUTIBZ, AUTIASP, AUTIBSP, AUTIA1716, AUTIB1716)>;
+def : InstRW<[Ampere1Write_8cyc_1BS_1A],
+        (instregex "BRA(A|AZ|B|BZ)", "RETA(A|B)", "ERETA(A|B)")>;
+def : InstRW<[Ampere1Write_8cyc_1BS_2A],
+        (instrs BLRAA, BLRAAZ, BLRAB, BLRABZ)>;
+//def : InstRW<[Ampere1Write_7cyc_1BS],
+//	(instrs PACIAZ, PACIBZ, PACIASP, PACIBSP, PACIA1716, PACIB1716)>;
+def : InstRW<[Ampere1Write_11cyc_1BS_1L], (instregex "^LDRA(A|B)")>;
+def : InstRW<[Ampere1Write_7cyc_1BS], (instrs XPACD, XPACI)>;
+
+// Vector integer instructions
+// -- absolute difference
+def : InstRW<[Ampere1Write_3cyc_1XY],
+             (instregex "^SABAv", "^SABALv", "^SABDv", "^SABDLv",
+                        "^UABAv", "^UABALv", "^UABDv", "^UABDLv")>;
+// -- arithmetic
+def : InstRW<[Ampere1Write_3cyc_1XY],
+        (instregex "^ABSv", "^(ADD|SUB)v", "^SADDLv", "^SADDW", "SHADD",
+                   "SHSUB", "^SRHADD", "^URHADD", "SSUBL", "SSUBW",
+                   "^UADDLv", "^UADDW", "UHADD", "UHSUB", "USUBL", "USUBW")>;
+// -- arithmetic, horizontal, 16B
+def : InstRW<[Ampere1Write_12cyc_4XY],
+            (instregex "^ADDVv16i8v", "^SADDLVv16i8v", "^UADDLVv16i8v")>;
+def : InstRW<[Ampere1Write_12cyc_4XY],
+            (instregex "^[SU](MIN|MAX)Vv16i8v")>;
+// -- arithmetic, horizontal, 4H/4S
+def : InstRW<[Ampere1Write_6cyc_2XY],
+            (instregex "^[SU]?ADDL?V(v8i8|v4i16|v2i32)v")>;
+def : InstRW<[Ampere1Write_6cyc_2XY],
+            (instregex "^[SU](MIN|MAX)V(v4i16|v4i32)v")>;
+// -- arithmetic, horizontal, 8B/8H
+def : InstRW<[Ampere1Write_9cyc_3XY],
+            (instregex "^[SU]?ADDL?V(v8i16|v4i32)v")>;
+def : InstRW<[Ampere1Write_9cyc_3XY],
+            (instregex "^[SU](MIN|MAX)V(v8i8|v8i16)v")>;
+// -- arithmetic, narrowing
+def : InstRW<[Ampere1Write_5cyc_2XY], (instregex "(ADD|SUB)HNv.*")>;
+def : InstRW<[Ampere1Write_5cyc_2XY], (instregex "(RADD|RSUB)HNv.*")>;
+// -- arithmetic, pairwise
+def : InstRW<[Ampere1Write_3cyc_1XY],
+        (instregex "^ADDPv", "^SADALP", "^UADALP", "^SADDLPv", "^UADDLPv")>;
+// -- arithmetic, saturating
+def : InstRW<[Ampere1Write_3cyc_1XY],
+        (instregex "^SQADD", "^SQSUB", "^SUQADD", "^UQADD", "^UQSUB", "^USQADD")>;
+// -- bit count
+def : InstRW<[Ampere1Write_2cyc_1XY],
+        (instregex "^(CLS|CLZ|CNT)v")>;
+// -- compare
+def : InstRW<[Ampere1Write_3cyc_1XY],
+        (instregex "^CMEQv", "^CMGEv", "^CMGTv", "^CMLEv", "^CMLTv",
+                   "^CMHIv", "^CMHSv")>;
+// -- compare non-zero
+def : InstRW<[Ampere1Write_2cyc_1XY], (instregex "^CMTSTv")>;
+// -- dot product
+def : InstRW<[Ampere1Write_3cyc_1XY], (instregex "^(S|SU|U|US)DOTv")>;
+// -- fp reciprocal estimate
+def : InstRW<[Ampere1Write_5cyc_1XY], (instregex "^FRECPEv", "^FRSQRTEv")>;
+// -- integer reciprocal estimate
+def : InstRW<[Ampere1Write_5cyc_1XY], (instregex "^URECPEv", "^URSQRTEv")>;
+// -- logical
+def : InstRW<[Ampere1Write_2cyc_1XY],
+        (instregex "^ANDv", "^BICv", "^EORv", "^ORRv", "^ORNv", "^NOTv")>;
+// -- logical, narrowing
+def : InstRW<[Ampere1Write_5cyc_2XY],
+        (instregex "RSHRNv",
+                   "SHRNv", "SQSHRNv", "SQSHRUNv",
+                   "UQXTNv")>;
+// -- matrix multiply
+def : InstRW<[Ampere1Write_6cyc_2XY],
+        (instrs SMMLA, UMMLA, USMMLA)>;
+// -- max/min
+def : InstRW<[Ampere1Write_3cyc_1XY],
+        (instregex "^SMAXv", "^SMINv", "^UMAXv", "^UMINv")>;
+def : InstRW<[Ampere1Write_3cyc_1XY],
+        (instregex "^SMAXPv", "^SMINPv", "^UMAXPv", "^UMINPv")>;
+// -- move immediate
+def : InstRW<[Ampere1Write_2cyc_1XY], (instregex "^MOVIv", "^MVNIv")>;
+// -- multiply
+def : InstRW<[Ampere1Write_3cyc_1XY],
+        (instregex "MULv", "SMULLv", "UMULLv", "SQDMUL(H|L)v", "SQRDMULHv")>;
+// -- multiply accumulate
+def : InstRW<[Ampere1Write_3cyc_1XY],
+        (instregex "MLAv", "MLSv", "(S|U|SQD)(MLAL|MLSL)v", "SQRDML(A|S)Hv")>;
+// -- negation, saturating
+def : InstRW<[Ampere1Write_2cyc_1XY], (instregex "^SQABS", "^SQNEG")>;
+// -- reverse bits/bytes
+def : InstRW<[Ampere1Write_2cyc_1XY],
+        (instregex "^RBITv", "^REV16v", "^REV32v", "^REV64v")>;
+// -- shift
+def : InstRW<[Ampere1Write_3cyc_1XY], (instregex "^[SU]SHL(v16i8|v8i16|v4i32|v2i64)")>;
+// -- shift and accumulate
+def : InstRW<[Ampere1Write_3cyc_1XY],
+        (instregex "SRSRAv", "SSRAv", "URSRAv", "USRAv")>;
+// -- shift, saturating
+def : InstRW<[Ampere1Write_3cyc_1XY],
+        (instregex "^SQRSHLv", "^SQRSHRNv", "^SQRSHRUNv", "^SQSHL", "^SQSHLU",
+                   "^SQXTNv", "^SQXTUNv", "^UQSHRNv", "UQRSHRNv", "^UQRSHL",
+                   "^UQSHL")>;
+
+// Vector miscellaneous instructions
+// -- duplicate element
+def : InstRW<[Ampere1Write_2cyc_1XY], (instregex "^DUPv.+lane")>;
+// -- duplicate from GPR
+def : InstRW<[Ampere1Write_5cyc_1BS], (instregex "^DUPv.+gpr")>;
+// -- extract narrow
+def : InstRW<[Ampere1Write_2cyc_1XY], (instregex "^XTNv")>;
+// -- insert/extract element
+def : InstRW<[Ampere1Write_2cyc_1XY], (instregex "^EXTv", "^INSv.+lane")>;
+// -- move FP immediate
+def : InstRW<[Ampere1Write_2cyc_1XY], (instregex "^FMOVv")>;
+// -- move element to GPR
+def : InstRW<[Ampere1Write_6cyc_1XY_1Z], (instregex "(S|U)MOVv")>;
+// -- move from GPR to any element
+def : InstRW<[Ampere1Write_7cyc_1BS_1XY], (instregex "^INSv.+gpr")>;
+// -- table lookup
+def : InstRW<[Ampere1Write_2cyc_1XY],
+            (instrs TBLv8i8One, TBLv16i8One, TBXv8i8One, TBXv16i8One)>;
+def : InstRW<[Ampere1Write_4cyc_2XY],
+            (instrs TBLv8i8Two, TBLv16i8Two, TBXv8i8Two, TBXv16i8Two)>;
+def : InstRW<[Ampere1Write_6cyc_3XY],
+            (instrs TBLv8i8Three, TBLv16i8Three, TBXv8i8Three, TBXv16i8Three)>;
+def : InstRW<[Ampere1Write_8cyc_4XY],
+            (instrs TBLv8i8Four, TBLv16i8Four, TBXv8i8Four, TBXv16i8Four)>;
+// -- transpose
+def : InstRW<[Ampere1Write_2cyc_1XY],
+              (instregex "^TRN1v", "^TRN2v", "^UZP1v", "^UZP2v")>;
+// -- zip/unzip
+def : InstRW<[Ampere1Write_2cyc_1XY], (instregex "^ZIP1v", "^ZIP2v")>;
+
+} // SchedModel = Ampere1Model
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64SchedPredAmpere.td b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64SchedPredAmpere.td
new file mode 100644
index 000000000000..8552c07bda56
--- /dev/null
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64SchedPredAmpere.td
@@ -0,0 +1,25 @@
+//===- AArch64SchedPredAmpere.td - AArch64 Sched Preds -----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines scheduling predicate definitions that are used by the
+// AArch64 Ampere Computing processors.
+//
+//===----------------------------------------------------------------------===//
+
+// Auxiliary predicates.
+
+// Check for a LSL shift <= 4
+def AmpereCheapLSL : MCSchedPredicate<
+                                CheckAny<[CheckShiftBy0,
+                                 CheckAll<
+                                   [CheckShiftLSL,
+                                    CheckAny<
+                                      [CheckShiftBy1,
+                                       CheckShiftBy2,
+                                       CheckShiftBy3,
+                                       CheckShiftBy4]>]>]>>;
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64SchedPredicates.td b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64SchedPredicates.td
index fc13b23b4cf8..19a3780c7381 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64SchedPredicates.td
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64SchedPredicates.td
@@ -1,441 +1,441 @@
 //===- AArch64SchedPredicates.td - AArch64 Sched Preds -----*- tablegen -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines scheduling predicate definitions that are used by the
 // AArch64 subtargets.
 //
 //===----------------------------------------------------------------------===//
 
 // Function mappers.
 
 // Check the extension type in arithmetic instructions.
 let FunctionMapper = "AArch64_AM::getArithExtendType" in {
   def CheckExtUXTB                      : CheckImmOperand_s<3, "AArch64_AM::UXTB">;
   def CheckExtUXTH                      : CheckImmOperand_s<3, "AArch64_AM::UXTH">;
   def CheckExtUXTW                      : CheckImmOperand_s<3, "AArch64_AM::UXTW">;
   def CheckExtUXTX                      : CheckImmOperand_s<3, "AArch64_AM::UXTX">;
   def CheckExtSXTB                      : CheckImmOperand_s<3, "AArch64_AM::SXTB">;
   def CheckExtSXTH                      : CheckImmOperand_s<3, "AArch64_AM::SXTH">;
   def CheckExtSXTW                      : CheckImmOperand_s<3, "AArch64_AM::SXTW">;
   def CheckExtSXTX                      : CheckImmOperand_s<3, "AArch64_AM::SXTX">;
 }
 
 // Check for shifting in extended arithmetic instructions.
 foreach I = {0-3} in {
   let FunctionMapper = "AArch64_AM::getArithShiftValue" in
   def CheckExtBy#I                      : CheckImmOperand<3, I>;
 }
 
 // Check the extension type in the register offset addressing mode.
 let FunctionMapper = "AArch64_AM::getMemExtendType" in {
   def CheckMemExtUXTW                   : CheckImmOperand_s<3, "AArch64_AM::UXTW">;
   def CheckMemExtLSL                    : CheckImmOperand_s<3, "AArch64_AM::UXTX">;
   def CheckMemExtSXTW                   : CheckImmOperand_s<3, "AArch64_AM::SXTW">;
   def CheckMemExtSXTX                   : CheckImmOperand_s<3, "AArch64_AM::SXTX">;
 }
 
 // Check for scaling in the register offset addressing mode.
 let FunctionMapper = "AArch64_AM::getMemDoShift" in
 def CheckMemScaled                      : CheckImmOperandSimple<4>;
 
 // Check the shifting type in arithmetic and logic instructions.
 let FunctionMapper = "AArch64_AM::getShiftType" in {
   def CheckShiftLSL                : CheckImmOperand_s<3, "AArch64_AM::LSL">;
   def CheckShiftLSR                : CheckImmOperand_s<3, "AArch64_AM::LSR">;
   def CheckShiftASR                : CheckImmOperand_s<3, "AArch64_AM::ASR">;
   def CheckShiftROR                : CheckImmOperand_s<3, "AArch64_AM::ROR">;
   def CheckShiftMSL                : CheckImmOperand_s<3, "AArch64_AM::MSL">;
 }
 
 // Check for shifting in arithmetic and logic instructions.
-foreach I = {0-3, 8} in {
+foreach I = {0-4, 8} in {
   let FunctionMapper = "AArch64_AM::getShiftValue" in
   def CheckShiftBy#I        : CheckImmOperand<3, I>;
 }
 
 // Generic predicates.
 
 // Identify whether an instruction is the 16-bit NEON form based on its result.
 def CheckHForm             : CheckAll<[CheckIsRegOperand<0>,
                                        CheckAny<[CheckRegOperand<0, H0>,
                                                  CheckRegOperand<0, H1>,
                                                  CheckRegOperand<0, H2>,
                                                  CheckRegOperand<0, H3>,
                                                  CheckRegOperand<0, H4>,
                                                  CheckRegOperand<0, H5>,
                                                  CheckRegOperand<0, H6>,
                                                  CheckRegOperand<0, H7>,
                                                  CheckRegOperand<0, H8>,
                                                  CheckRegOperand<0, H9>,
                                                  CheckRegOperand<0, H10>,
                                                  CheckRegOperand<0, H11>,
                                                  CheckRegOperand<0, H12>,
                                                  CheckRegOperand<0, H13>,
                                                  CheckRegOperand<0, H14>,
                                                  CheckRegOperand<0, H15>,
                                                  CheckRegOperand<0, H16>,
                                                  CheckRegOperand<0, H17>,
                                                  CheckRegOperand<0, H18>,
                                                  CheckRegOperand<0, H19>,
                                                  CheckRegOperand<0, H20>,
                                                  CheckRegOperand<0, H21>,
                                                  CheckRegOperand<0, H22>,
                                                  CheckRegOperand<0, H23>,
                                                  CheckRegOperand<0, H24>,
                                                  CheckRegOperand<0, H25>,
                                                  CheckRegOperand<0, H26>,
                                                  CheckRegOperand<0, H27>,
                                                  CheckRegOperand<0, H28>,
                                                  CheckRegOperand<0, H29>,
                                                  CheckRegOperand<0, H30>,
                                                  CheckRegOperand<0, H31>]>]>;
 
 // Identify whether an instruction is the 32-bit NEON form based on its result.
 def CheckSForm             : CheckAll<[CheckIsRegOperand<0>,
                                        CheckAny<[CheckRegOperand<0, S0>,
                                                  CheckRegOperand<0, S1>,
                                                  CheckRegOperand<0, S2>,
                                                  CheckRegOperand<0, S3>,
                                                  CheckRegOperand<0, S4>,
                                                  CheckRegOperand<0, S5>,
                                                  CheckRegOperand<0, S6>,
                                                  CheckRegOperand<0, S7>,
                                                  CheckRegOperand<0, S8>,
                                                  CheckRegOperand<0, S9>,
                                                  CheckRegOperand<0, S10>,
                                                  CheckRegOperand<0, S11>,
                                                  CheckRegOperand<0, S12>,
                                                  CheckRegOperand<0, S13>,
                                                  CheckRegOperand<0, S14>,
                                                  CheckRegOperand<0, S15>,
                                                  CheckRegOperand<0, S16>,
                                                  CheckRegOperand<0, S17>,
                                                  CheckRegOperand<0, S18>,
                                                  CheckRegOperand<0, S19>,
                                                  CheckRegOperand<0, S20>,
                                                  CheckRegOperand<0, S21>,
                                                  CheckRegOperand<0, S22>,
                                                  CheckRegOperand<0, S23>,
                                                  CheckRegOperand<0, S24>,
                                                  CheckRegOperand<0, S25>,
                                                  CheckRegOperand<0, S26>,
                                                  CheckRegOperand<0, S27>,
                                                  CheckRegOperand<0, S28>,
                                                  CheckRegOperand<0, S29>,
                                                  CheckRegOperand<0, S30>,
                                                  CheckRegOperand<0, S31>]>]>;
 
 // Identify whether an instruction is the 64-bit NEON form based on its result.
 def CheckDForm             : CheckAll<[CheckIsRegOperand<0>,
                                        CheckAny<[CheckRegOperand<0, D0>,
                                                  CheckRegOperand<0, D1>,
                                                  CheckRegOperand<0, D2>,
                                                  CheckRegOperand<0, D3>,
                                                  CheckRegOperand<0, D4>,
                                                  CheckRegOperand<0, D5>,
                                                  CheckRegOperand<0, D6>,
                                                  CheckRegOperand<0, D7>,
                                                  CheckRegOperand<0, D8>,
                                                  CheckRegOperand<0, D9>,
                                                  CheckRegOperand<0, D10>,
                                                  CheckRegOperand<0, D11>,
                                                  CheckRegOperand<0, D12>,
                                                  CheckRegOperand<0, D13>,
                                                  CheckRegOperand<0, D14>,
                                                  CheckRegOperand<0, D15>,
                                                  CheckRegOperand<0, D16>,
                                                  CheckRegOperand<0, D17>,
                                                  CheckRegOperand<0, D18>,
                                                  CheckRegOperand<0, D19>,
                                                  CheckRegOperand<0, D20>,
                                                  CheckRegOperand<0, D21>,
                                                  CheckRegOperand<0, D22>,
                                                  CheckRegOperand<0, D23>,
                                                  CheckRegOperand<0, D24>,
                                                  CheckRegOperand<0, D25>,
                                                  CheckRegOperand<0, D26>,
                                                  CheckRegOperand<0, D27>,
                                                  CheckRegOperand<0, D28>,
                                                  CheckRegOperand<0, D29>,
                                                  CheckRegOperand<0, D30>,
                                                  CheckRegOperand<0, D31>]>]>;
 
 // Identify whether an instruction is the 128-bit NEON form based on its result.
 def CheckQForm             : CheckAll<[CheckIsRegOperand<0>,
                                        CheckAny<[CheckRegOperand<0, Q0>,
                                                  CheckRegOperand<0, Q1>,
                                                  CheckRegOperand<0, Q2>,
                                                  CheckRegOperand<0, Q3>,
                                                  CheckRegOperand<0, Q4>,
                                                  CheckRegOperand<0, Q5>,
                                                  CheckRegOperand<0, Q6>,
                                                  CheckRegOperand<0, Q7>,
                                                  CheckRegOperand<0, Q8>,
                                                  CheckRegOperand<0, Q9>,
                                                  CheckRegOperand<0, Q10>,
                                                  CheckRegOperand<0, Q11>,
                                                  CheckRegOperand<0, Q12>,
                                                  CheckRegOperand<0, Q13>,
                                                  CheckRegOperand<0, Q14>,
                                                  CheckRegOperand<0, Q15>,
                                                  CheckRegOperand<0, Q16>,
                                                  CheckRegOperand<0, Q17>,
                                                  CheckRegOperand<0, Q18>,
                                                  CheckRegOperand<0, Q19>,
                                                  CheckRegOperand<0, Q20>,
                                                  CheckRegOperand<0, Q21>,
                                                  CheckRegOperand<0, Q22>,
                                                  CheckRegOperand<0, Q23>,
                                                  CheckRegOperand<0, Q24>,
                                                  CheckRegOperand<0, Q25>,
                                                  CheckRegOperand<0, Q26>,
                                                  CheckRegOperand<0, Q27>,
                                                  CheckRegOperand<0, Q28>,
                                                  CheckRegOperand<0, Q29>,
                                                  CheckRegOperand<0, Q30>,
                                                  CheckRegOperand<0, Q31>]>]>;
 
 // Identify arithmetic instructions with extend.
 def IsArithExtOp           : CheckOpcode<[ADDWrx, ADDXrx, ADDSWrx, ADDSXrx,
                                           SUBWrx, SUBXrx, SUBSWrx, SUBSXrx,
                                           ADDXrx64, ADDSXrx64,
                                           SUBXrx64, SUBSXrx64]>;
 
 // Identify arithmetic immediate instructions.
 def IsArithImmOp           : CheckOpcode<[ADDWri, ADDXri, ADDSWri, ADDSXri,
                                           SUBWri, SUBXri, SUBSWri, SUBSXri]>;
 
 // Identify arithmetic instructions with shift.
 def IsArithShiftOp         : CheckOpcode<[ADDWrs, ADDXrs, ADDSWrs, ADDSXrs,
                                           SUBWrs, SUBXrs, SUBSWrs, SUBSXrs]>;
 
 // Identify arithmetic instructions without shift.
 def IsArithUnshiftOp       : CheckOpcode<[ADDWrr, ADDXrr, ADDSWrr, ADDSXrr,
                                           SUBWrr, SUBXrr, SUBSWrr, SUBSXrr]>;
 
 // Identify logic immediate instructions.
 def IsLogicImmOp           : CheckOpcode<[ANDWri, ANDXri,
                                           EORWri, EORXri,
                                           ORRWri, ORRXri]>;
 
 // Identify logic instructions with shift.
 def IsLogicShiftOp         : CheckOpcode<[ANDWrs, ANDXrs, ANDSWrs, ANDSXrs,
                                           BICWrs, BICXrs, BICSWrs, BICSXrs,
                                           EONWrs, EONXrs,
                                           EORWrs, EORXrs,
                                           ORNWrs, ORNXrs,
                                           ORRWrs, ORRXrs]>;
 
 // Identify logic instructions without shift.
 def IsLogicUnshiftOp       : CheckOpcode<[ANDWrr, ANDXrr, ANDSWrr, ANDSXrr,
                                           BICWrr, BICXrr, BICSWrr, BICSXrr,
                                           EONWrr, EONXrr,
                                           EORWrr, EORXrr,
                                           ORNWrr, ORNXrr,
                                           ORRWrr, ORRXrr]>;
 
 // Identify arithmetic and logic immediate instructions.
 def IsArithLogicImmOp      : CheckOpcode<!listconcat(IsArithImmOp.ValidOpcodes,
                                                      IsLogicImmOp.ValidOpcodes)>;
 
 // Identify arithmetic and logic instructions with shift.
 def IsArithLogicShiftOp    : CheckOpcode<!listconcat(IsArithShiftOp.ValidOpcodes,
                                                      IsLogicShiftOp.ValidOpcodes)>;
 
 // Identify arithmetic and logic instructions without shift.
 def IsArithLogicUnshiftOp  : CheckOpcode<!listconcat(IsArithUnshiftOp.ValidOpcodes,
                                                      IsLogicUnshiftOp.ValidOpcodes)>;
 
 // Identify whether an instruction is an ASIMD
 // load using the post index addressing mode.
 def IsLoadASIMDPostOp      : CheckOpcode<[LD1Onev8b_POST, LD1Onev4h_POST, LD1Onev2s_POST, LD1Onev1d_POST,
                                           LD1Onev16b_POST, LD1Onev8h_POST, LD1Onev4s_POST, LD1Onev2d_POST,
                                           LD1Twov8b_POST, LD1Twov4h_POST, LD1Twov2s_POST, LD1Twov1d_POST,
                                           LD1Twov16b_POST, LD1Twov8h_POST, LD1Twov4s_POST, LD1Twov2d_POST,
                                           LD1Threev8b_POST, LD1Threev4h_POST, LD1Threev2s_POST, LD1Threev1d_POST,
                                           LD1Threev16b_POST, LD1Threev8h_POST, LD1Threev4s_POST, LD1Threev2d_POST,
                                           LD1Fourv8b_POST, LD1Fourv4h_POST, LD1Fourv2s_POST, LD1Fourv1d_POST,
                                           LD1Fourv16b_POST, LD1Fourv8h_POST, LD1Fourv4s_POST, LD1Fourv2d_POST,
                                           LD1i8_POST, LD1i16_POST, LD1i32_POST, LD1i64_POST,
                                           LD1Rv8b_POST, LD1Rv4h_POST, LD1Rv2s_POST, LD1Rv1d_POST,
                                           LD1Rv16b_POST, LD1Rv8h_POST, LD1Rv4s_POST, LD1Rv2d_POST,
                                           LD2Twov8b_POST, LD2Twov4h_POST, LD2Twov2s_POST,
                                           LD2Twov16b_POST, LD2Twov8h_POST, LD2Twov4s_POST, LD2Twov2d_POST,
                                           LD2i8_POST, LD2i16_POST, LD2i32_POST, LD2i64_POST,
                                           LD2Rv8b_POST, LD2Rv4h_POST, LD2Rv2s_POST, LD2Rv1d_POST,
                                           LD2Rv16b_POST, LD2Rv8h_POST, LD2Rv4s_POST, LD2Rv2d_POST,
                                           LD3Threev8b_POST, LD3Threev4h_POST, LD3Threev2s_POST,
                                           LD3Threev16b_POST, LD3Threev8h_POST, LD3Threev4s_POST, LD3Threev2d_POST,
                                           LD3i8_POST, LD3i16_POST, LD3i32_POST, LD3i64_POST,
                                           LD3Rv8b_POST, LD3Rv4h_POST, LD3Rv2s_POST, LD3Rv1d_POST,
                                           LD3Rv16b_POST, LD3Rv8h_POST, LD3Rv4s_POST, LD3Rv2d_POST,
                                           LD4Fourv8b_POST, LD4Fourv4h_POST, LD4Fourv2s_POST,
                                           LD4Fourv16b_POST, LD4Fourv8h_POST, LD4Fourv4s_POST, LD4Fourv2d_POST,
                                           LD4i8_POST, LD4i16_POST, LD4i32_POST, LD4i64_POST,
                                           LD4Rv8b_POST, LD4Rv4h_POST, LD4Rv2s_POST, LD4Rv1d_POST,
                                           LD4Rv16b_POST, LD4Rv8h_POST, LD4Rv4s_POST, LD4Rv2d_POST]>;
 
 // Identify whether an instruction is an ASIMD
 // store using the post index addressing mode.
 def IsStoreASIMDPostOp     : CheckOpcode<[ST1Onev8b_POST, ST1Onev4h_POST, ST1Onev2s_POST, ST1Onev1d_POST,
                                           ST1Onev16b_POST, ST1Onev8h_POST, ST1Onev4s_POST, ST1Onev2d_POST,
                                           ST1Twov8b_POST, ST1Twov4h_POST, ST1Twov2s_POST, ST1Twov1d_POST,
                                           ST1Twov16b_POST, ST1Twov8h_POST, ST1Twov4s_POST, ST1Twov2d_POST,
                                           ST1Threev8b_POST, ST1Threev4h_POST, ST1Threev2s_POST, ST1Threev1d_POST,
                                           ST1Threev16b_POST, ST1Threev8h_POST, ST1Threev4s_POST, ST1Threev2d_POST,
                                           ST1Fourv8b_POST, ST1Fourv4h_POST, ST1Fourv2s_POST, ST1Fourv1d_POST,
                                           ST1Fourv16b_POST, ST1Fourv8h_POST, ST1Fourv4s_POST, ST1Fourv2d_POST,
                                           ST1i8_POST, ST1i16_POST, ST1i32_POST, ST1i64_POST,
                                           ST2Twov8b_POST, ST2Twov4h_POST, ST2Twov2s_POST,
                                           ST2Twov16b_POST, ST2Twov8h_POST, ST2Twov4s_POST, ST2Twov2d_POST,
                                           ST2i8_POST, ST2i16_POST, ST2i32_POST, ST2i64_POST,
                                           ST3Threev8b_POST, ST3Threev4h_POST, ST3Threev2s_POST,
                                           ST3Threev16b_POST, ST3Threev8h_POST, ST3Threev4s_POST, ST3Threev2d_POST,
                                           ST3i8_POST, ST3i16_POST, ST3i32_POST, ST3i64_POST,
                                           ST4Fourv8b_POST, ST4Fourv4h_POST, ST4Fourv2s_POST,
                                           ST4Fourv16b_POST, ST4Fourv8h_POST, ST4Fourv4s_POST, ST4Fourv2d_POST,
                                           ST4i8_POST, ST4i16_POST, ST4i32_POST, ST4i64_POST]>;
 
 // Identify whether an instruction is an ASIMD load
 // or store using the post index addressing mode.
 def IsLoadStoreASIMDPostOp : CheckOpcode<!listconcat(IsLoadASIMDPostOp.ValidOpcodes,
                                                      IsStoreASIMDPostOp.ValidOpcodes)>;
 
 // Identify whether an instruction is a load
 // using the register offset addressing mode.
 def IsLoadRegOffsetOp      : CheckOpcode<[PRFMroW, PRFMroX,
                                           LDRBBroW, LDRBBroX,
                                           LDRSBWroW, LDRSBWroX, LDRSBXroW, LDRSBXroX,
                                           LDRHHroW, LDRHHroX,
                                           LDRSHWroW, LDRSHWroX, LDRSHXroW, LDRSHXroX,
                                           LDRWroW, LDRWroX,
                                           LDRSWroW, LDRSWroX,
                                           LDRXroW, LDRXroX,
                                           LDRBroW, LDRBroX,
                                           LDRHroW, LDRHroX,
                                           LDRSroW, LDRSroX,
                                           LDRDroW, LDRDroX,
                                           LDRQroW, LDRQroX]>;
 
 // Identify whether an instruction is a store
 // using the register offset addressing mode.
 def IsStoreRegOffsetOp     : CheckOpcode<[STRBBroW, STRBBroX,
                                           STRHHroW, STRHHroX,
                                           STRWroW, STRWroX,
                                           STRXroW, STRXroX,
                                           STRBroW, STRBroX,
                                           STRHroW, STRHroX,
                                           STRSroW, STRSroX,
                                           STRDroW, STRDroX,
                                           STRQroW, STRQroX]>;
 
 // Identify whether an instruction is a load or
 // store using the register offset addressing mode.
 def IsLoadStoreRegOffsetOp : CheckOpcode<!listconcat(IsLoadRegOffsetOp.ValidOpcodes,
                                                      IsStoreRegOffsetOp.ValidOpcodes)>;
 
 // Target predicates.
 
 // Identify an instruction that effectively transfers a register to another.
 def IsCopyIdiomFn     : TIIPredicate<"isCopyIdiom",
                                      MCOpcodeSwitchStatement<
                                        [// MOV {Rd, SP}, {SP, Rn} =>
                                         // ADD {Rd, SP}, {SP, Rn}, #0
                                         MCOpcodeSwitchCase<
                                           [ADDWri, ADDXri],
                                           MCReturnStatement<
                                             CheckAll<
                                               [CheckIsRegOperand<0>,
                                                CheckIsRegOperand<1>,
                                                CheckAny<
                                                  [CheckRegOperand<0, WSP>,
                                                   CheckRegOperand<0, SP>,
                                                   CheckRegOperand<1, WSP>,
                                                   CheckRegOperand<1, SP>]>,
                                                CheckZeroOperand<2>]>>>,
                                         // MOV Rd, Rm =>
                                         // ORR Rd, ZR, Rm, LSL #0
                                         MCOpcodeSwitchCase<
                                           [ORRWrs, ORRXrs],
                                           MCReturnStatement<
                                             CheckAll<
                                               [CheckIsRegOperand<1>,
                                                CheckIsRegOperand<2>,
                                                CheckAny<
                                                  [CheckRegOperand<1, WZR>,
                                                   CheckRegOperand<1, XZR>]>,
                                                CheckShiftBy0]>>>],
                                        MCReturnStatement<FalsePred>>>;
 def IsCopyIdiomPred   : MCSchedPredicate<IsCopyIdiomFn>;
 
 // Identify arithmetic instructions with an extended register.
 def RegExtendedFn     : TIIPredicate<"hasExtendedReg",
                                      MCOpcodeSwitchStatement<
                                        [MCOpcodeSwitchCase<
                                          IsArithExtOp.ValidOpcodes,
                                          MCReturnStatement<
                                            CheckNot<CheckZeroOperand<3>>>>],
                                        MCReturnStatement<FalsePred>>>;
 def RegExtendedPred   : MCSchedPredicate<RegExtendedFn>;
 
 // Identify arithmetic and logic instructions with a shifted register.
 def RegShiftedFn      : TIIPredicate<"hasShiftedReg",
                                      MCOpcodeSwitchStatement<
                                        [MCOpcodeSwitchCase<
                                           IsArithLogicShiftOp.ValidOpcodes,
                                           MCReturnStatement<
                                             CheckNot<CheckZeroOperand<3>>>>],
                                        MCReturnStatement<FalsePred>>>;
 def RegShiftedPred    : MCSchedPredicate<RegShiftedFn>;
 
 // Identify a load or store using the register offset addressing mode
 // with an extended or scaled register.
 def ScaledIdxFn       : TIIPredicate<"isScaledAddr",
                                      MCOpcodeSwitchStatement<
                                        [MCOpcodeSwitchCase<
                                           IsLoadStoreRegOffsetOp.ValidOpcodes,
                                           MCReturnStatement<
                                             CheckAny<[CheckNot<CheckMemExtLSL>,
                                                       CheckMemScaled]>>>],
                                        MCReturnStatement<FalsePred>>>;
 def ScaledIdxPred     : MCSchedPredicate<ScaledIdxFn>;
 
 // Identify an instruction that effectively resets a FP register to zero.
 def IsZeroFPIdiomFn   : TIIPredicate<"isZeroFPIdiom",
                                      MCOpcodeSwitchStatement<
                                        [// MOVI Vd, #0
                                         MCOpcodeSwitchCase<
                                           [MOVIv8b_ns, MOVIv16b_ns,
                                            MOVID, MOVIv2d_ns],
                                           MCReturnStatement<CheckZeroOperand<1>>>,
                                         // MOVI Vd, #0, LSL #0
                                         MCOpcodeSwitchCase<
                                           [MOVIv4i16, MOVIv8i16,
                                            MOVIv2i32, MOVIv4i32],
                                           MCReturnStatement<
                                             CheckAll<
                                               [CheckZeroOperand<1>,
                                                CheckZeroOperand<2>]>>>],
                                        MCReturnStatement<FalsePred>>>;
 def IsZeroFPIdiomPred : MCSchedPredicate<IsZeroFPIdiomFn>;
 
 // Identify an instruction that effectively resets a GP register to zero.
 def IsZeroIdiomFn     : TIIPredicate<"isZeroIdiom",
                                     MCOpcodeSwitchStatement<
                                       [// ORR Rd, ZR, #0
                                        MCOpcodeSwitchCase<
                                          [ORRWri, ORRXri],
                                          MCReturnStatement<
                                            CheckAll<
                                              [CheckIsRegOperand<1>,
                                               CheckAny<
                                                 [CheckRegOperand<1, WZR>,
                                                  CheckRegOperand<1, XZR>]>,
                                               CheckZeroOperand<2>]>>>],
                                       MCReturnStatement<FalsePred>>>;
 def IsZeroIdiomPred   : MCSchedPredicate<IsZeroIdiomFn>;
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
index 8a7e20237271..9f2753584db8 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -1,377 +1,383 @@
 //===-- AArch64Subtarget.cpp - AArch64 Subtarget Information ----*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the AArch64 specific subclass of TargetSubtarget.
 //
 //===----------------------------------------------------------------------===//
 
 #include "AArch64Subtarget.h"
 
 #include "AArch64.h"
 #include "AArch64InstrInfo.h"
 #include "AArch64PBQPRegAlloc.h"
 #include "AArch64TargetMachine.h"
 #include "GISel/AArch64CallLowering.h"
 #include "GISel/AArch64LegalizerInfo.h"
 #include "GISel/AArch64RegisterBankInfo.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
 #include "llvm/CodeGen/MachineScheduler.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/Support/AArch64TargetParser.h"
 #include "llvm/Support/TargetParser.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "aarch64-subtarget"
 
 #define GET_SUBTARGETINFO_CTOR
 #define GET_SUBTARGETINFO_TARGET_DESC
 #include "AArch64GenSubtargetInfo.inc"
 
 static cl::opt<bool>
 EnableEarlyIfConvert("aarch64-early-ifcvt", cl::desc("Enable the early if "
                      "converter pass"), cl::init(true), cl::Hidden);
 
 // If OS supports TBI, use this flag to enable it.
 static cl::opt<bool>
 UseAddressTopByteIgnored("aarch64-use-tbi", cl::desc("Assume that top byte of "
                          "an address is ignored"), cl::init(false), cl::Hidden);
 
 static cl::opt<bool>
     UseNonLazyBind("aarch64-enable-nonlazybind",
                    cl::desc("Call nonlazybind functions via direct GOT load"),
                    cl::init(false), cl::Hidden);
 
 static cl::opt<bool> UseAA("aarch64-use-aa", cl::init(true),
                            cl::desc("Enable the use of AA during codegen."));
 
 AArch64Subtarget &AArch64Subtarget::initializeSubtargetDependencies(
     StringRef FS, StringRef CPUString, StringRef TuneCPUString) {
   // Determine default and user-specified characteristics
 
   if (CPUString.empty())
     CPUString = "generic";
 
   if (TuneCPUString.empty())
     TuneCPUString = CPUString;
 
   ParseSubtargetFeatures(CPUString, TuneCPUString, FS);
   initializeProperties();
 
   return *this;
 }
 
 void AArch64Subtarget::initializeProperties() {
   // Initialize CPU specific properties. We should add a tablegen feature for
   // this in the future so we can specify it together with the subtarget
   // features.
   switch (ARMProcFamily) {
   case Others:
     break;
   case Carmel:
     CacheLineSize = 64;
     break;
   case CortexA35:
     break;
   case CortexA53:
   case CortexA55:
     PrefFunctionLogAlignment = 4;
     break;
   case CortexA57:
     MaxInterleaveFactor = 4;
     PrefFunctionLogAlignment = 4;
     break;
   case CortexA65:
     PrefFunctionLogAlignment = 3;
     break;
   case CortexA72:
   case CortexA73:
   case CortexA75:
   case CortexA76:
   case CortexA77:
   case CortexA78:
   case CortexA78C:
   case CortexR82:
   case CortexX1:
   case CortexX1C:
     PrefFunctionLogAlignment = 4;
     break;
   case CortexA510:
   case CortexA710:
   case CortexX2:
     PrefFunctionLogAlignment = 4;
     VScaleForTuning = 1;
     break;
   case A64FX:
     CacheLineSize = 256;
     PrefFunctionLogAlignment = 3;
     PrefLoopLogAlignment = 2;
     MaxInterleaveFactor = 4;
     PrefetchDistance = 128;
     MinPrefetchStride = 1024;
     MaxPrefetchIterationsAhead = 4;
     VScaleForTuning = 4;
     break;
   case AppleA7:
   case AppleA10:
   case AppleA11:
   case AppleA12:
   case AppleA13:
   case AppleA14:
     CacheLineSize = 64;
     PrefetchDistance = 280;
     MinPrefetchStride = 2048;
     MaxPrefetchIterationsAhead = 3;
     break;
   case ExynosM3:
     MaxInterleaveFactor = 4;
     MaxJumpTableSize = 20;
     PrefFunctionLogAlignment = 5;
     PrefLoopLogAlignment = 4;
     break;
   case Falkor:
     MaxInterleaveFactor = 4;
     // FIXME: remove this to enable 64-bit SLP if performance looks good.
     MinVectorRegisterBitWidth = 128;
     CacheLineSize = 128;
     PrefetchDistance = 820;
     MinPrefetchStride = 2048;
     MaxPrefetchIterationsAhead = 8;
     break;
   case Kryo:
     MaxInterleaveFactor = 4;
     VectorInsertExtractBaseCost = 2;
     CacheLineSize = 128;
     PrefetchDistance = 740;
     MinPrefetchStride = 1024;
     MaxPrefetchIterationsAhead = 11;
     // FIXME: remove this to enable 64-bit SLP if performance looks good.
     MinVectorRegisterBitWidth = 128;
     break;
   case NeoverseE1:
     PrefFunctionLogAlignment = 3;
     break;
   case NeoverseN1:
     PrefFunctionLogAlignment = 4;
     PrefLoopLogAlignment = 5;
     MaxBytesForLoopAlignment = 16;
     break;
   case NeoverseN2:
     PrefFunctionLogAlignment = 4;
     PrefLoopLogAlignment = 5;
     MaxBytesForLoopAlignment = 16;
     VScaleForTuning = 1;
     break;
   case NeoverseV1:
     PrefFunctionLogAlignment = 4;
     PrefLoopLogAlignment = 5;
     MaxBytesForLoopAlignment = 16;
     VScaleForTuning = 2;
     break;
   case Neoverse512TVB:
     PrefFunctionLogAlignment = 4;
     VScaleForTuning = 1;
     MaxInterleaveFactor = 4;
     break;
   case Saphira:
     MaxInterleaveFactor = 4;
     // FIXME: remove this to enable 64-bit SLP if performance looks good.
     MinVectorRegisterBitWidth = 128;
     break;
   case ThunderX2T99:
     CacheLineSize = 64;
     PrefFunctionLogAlignment = 3;
     PrefLoopLogAlignment = 2;
     MaxInterleaveFactor = 4;
     PrefetchDistance = 128;
     MinPrefetchStride = 1024;
     MaxPrefetchIterationsAhead = 4;
     // FIXME: remove this to enable 64-bit SLP if performance looks good.
     MinVectorRegisterBitWidth = 128;
     break;
   case ThunderX:
   case ThunderXT88:
   case ThunderXT81:
   case ThunderXT83:
     CacheLineSize = 128;
     PrefFunctionLogAlignment = 3;
     PrefLoopLogAlignment = 2;
     // FIXME: remove this to enable 64-bit SLP if performance looks good.
     MinVectorRegisterBitWidth = 128;
     break;
   case TSV110:
     CacheLineSize = 64;
     PrefFunctionLogAlignment = 4;
     PrefLoopLogAlignment = 2;
     break;
   case ThunderX3T110:
     CacheLineSize = 64;
     PrefFunctionLogAlignment = 4;
     PrefLoopLogAlignment = 2;
     MaxInterleaveFactor = 4;
     PrefetchDistance = 128;
     MinPrefetchStride = 1024;
     MaxPrefetchIterationsAhead = 4;
     // FIXME: remove this to enable 64-bit SLP if performance looks good.
     MinVectorRegisterBitWidth = 128;
     break;
+  case Ampere1:
+    CacheLineSize = 64;
+    PrefFunctionLogAlignment = 6;
+    PrefLoopLogAlignment = 6;
+    MaxInterleaveFactor = 4;
+    break;
   }
 }
 
 AArch64Subtarget::AArch64Subtarget(const Triple &TT, const std::string &CPU,
                                    const std::string &TuneCPU,
                                    const std::string &FS,
                                    const TargetMachine &TM, bool LittleEndian,
                                    unsigned MinSVEVectorSizeInBitsOverride,
                                    unsigned MaxSVEVectorSizeInBitsOverride)
     : AArch64GenSubtargetInfo(TT, CPU, TuneCPU, FS),
       ReserveXRegister(AArch64::GPR64commonRegClass.getNumRegs()),
       CustomCallSavedXRegs(AArch64::GPR64commonRegClass.getNumRegs()),
       IsLittle(LittleEndian),
       MinSVEVectorSizeInBits(MinSVEVectorSizeInBitsOverride),
       MaxSVEVectorSizeInBits(MaxSVEVectorSizeInBitsOverride), TargetTriple(TT),
       InstrInfo(initializeSubtargetDependencies(FS, CPU, TuneCPU)),
       TLInfo(TM, *this) {
   if (AArch64::isX18ReservedByDefault(TT))
     ReserveXRegister.set(18);
 
   CallLoweringInfo.reset(new AArch64CallLowering(*getTargetLowering()));
   InlineAsmLoweringInfo.reset(new InlineAsmLowering(getTargetLowering()));
   Legalizer.reset(new AArch64LegalizerInfo(*this));
 
   auto *RBI = new AArch64RegisterBankInfo(*getRegisterInfo());
 
   // FIXME: At this point, we can't rely on Subtarget having RBI.
   // It's awkward to mix passing RBI and the Subtarget; should we pass
   // TII/TRI as well?
   InstSelector.reset(createAArch64InstructionSelector(
       *static_cast<const AArch64TargetMachine *>(&TM), *this, *RBI));
 
   RegBankInfo.reset(RBI);
 }
 
 const CallLowering *AArch64Subtarget::getCallLowering() const {
   return CallLoweringInfo.get();
 }
 
 const InlineAsmLowering *AArch64Subtarget::getInlineAsmLowering() const {
   return InlineAsmLoweringInfo.get();
 }
 
 InstructionSelector *AArch64Subtarget::getInstructionSelector() const {
   return InstSelector.get();
 }
 
 const LegalizerInfo *AArch64Subtarget::getLegalizerInfo() const {
   return Legalizer.get();
 }
 
 const RegisterBankInfo *AArch64Subtarget::getRegBankInfo() const {
   return RegBankInfo.get();
 }
 
 /// Find the target operand flags that describe how a global value should be
 /// referenced for the current subtarget.
 unsigned
 AArch64Subtarget::ClassifyGlobalReference(const GlobalValue *GV,
                                           const TargetMachine &TM) const {
   // MachO large model always goes via a GOT, simply to get a single 8-byte
   // absolute relocation on all global addresses.
   if (TM.getCodeModel() == CodeModel::Large && isTargetMachO())
     return AArch64II::MO_GOT;
 
   if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV)) {
     if (GV->hasDLLImportStorageClass())
       return AArch64II::MO_GOT | AArch64II::MO_DLLIMPORT;
     if (getTargetTriple().isOSWindows())
       return AArch64II::MO_GOT | AArch64II::MO_COFFSTUB;
     return AArch64II::MO_GOT;
   }
 
   // The small code model's direct accesses use ADRP, which cannot
   // necessarily produce the value 0 (if the code is above 4GB).
   // Same for the tiny code model, where we have a pc relative LDR.
   if ((useSmallAddressing() || TM.getCodeModel() == CodeModel::Tiny) &&
       GV->hasExternalWeakLinkage())
     return AArch64II::MO_GOT;
 
   // References to tagged globals are marked with MO_NC | MO_TAGGED to indicate
   // that their nominal addresses are tagged and outside of the code model. In
   // AArch64ExpandPseudo::expandMI we emit an additional instruction to set the
   // tag if necessary based on MO_TAGGED.
   if (AllowTaggedGlobals && !isa<FunctionType>(GV->getValueType()))
     return AArch64II::MO_NC | AArch64II::MO_TAGGED;
 
   return AArch64II::MO_NO_FLAG;
 }
 
 unsigned AArch64Subtarget::classifyGlobalFunctionReference(
     const GlobalValue *GV, const TargetMachine &TM) const {
   // MachO large model always goes via a GOT, because we don't have the
   // relocations available to do anything else..
   if (TM.getCodeModel() == CodeModel::Large && isTargetMachO() &&
       !GV->hasInternalLinkage())
     return AArch64II::MO_GOT;
 
   // NonLazyBind goes via GOT unless we know it's available locally.
   auto *F = dyn_cast<Function>(GV);
   if (UseNonLazyBind && F && F->hasFnAttribute(Attribute::NonLazyBind) &&
       !TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
     return AArch64II::MO_GOT;
 
   // Use ClassifyGlobalReference for setting MO_DLLIMPORT/MO_COFFSTUB.
   if (getTargetTriple().isOSWindows())
     return ClassifyGlobalReference(GV, TM);
 
   return AArch64II::MO_NO_FLAG;
 }
 
 void AArch64Subtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
                                            unsigned NumRegionInstrs) const {
   // LNT run (at least on Cyclone) showed reasonably significant gains for
   // bi-directional scheduling. 253.perlbmk.
   Policy.OnlyTopDown = false;
   Policy.OnlyBottomUp = false;
   // Enabling or Disabling the latency heuristic is a close call: It seems to
   // help nearly no benchmark on out-of-order architectures, on the other hand
   // it regresses register pressure on a few benchmarking.
   Policy.DisableLatencyHeuristic = DisableLatencySchedHeuristic;
 }
 
 bool AArch64Subtarget::enableEarlyIfConversion() const {
   return EnableEarlyIfConvert;
 }
 
 bool AArch64Subtarget::supportsAddressTopByteIgnored() const {
   if (!UseAddressTopByteIgnored)
     return false;
 
   if (TargetTriple.isiOS()) {
     return TargetTriple.getiOSVersion() >= VersionTuple(8);
   }
 
   return false;
 }
 
 std::unique_ptr<PBQPRAConstraint>
 AArch64Subtarget::getCustomPBQPConstraints() const {
   return balanceFPOps() ? std::make_unique<A57ChainingConstraint>() : nullptr;
 }
 
 void AArch64Subtarget::mirFileLoaded(MachineFunction &MF) const {
   // We usually compute max call frame size after ISel. Do the computation now
   // if the .mir file didn't specify it. Note that this will probably give you
   // bogus values after PEI has eliminated the callframe setup/destroy pseudo
   // instructions, specify explicitly if you need it to be correct.
   MachineFrameInfo &MFI = MF.getFrameInfo();
   if (!MFI.isMaxCallFrameSizeComputed())
     MFI.computeMaxCallFrameSize(MF);
 }
 
 bool AArch64Subtarget::useAA() const { return UseAA; }
diff --git a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.h b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.h
index 061db926ee2b..1c14e0a55049 100644
--- a/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/contrib/llvm-project/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -1,697 +1,698 @@
 //===--- AArch64Subtarget.h - Define Subtarget for the AArch64 -*- C++ -*--===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file declares the AArch64 specific subclass of TargetSubtarget.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIB_TARGET_AARCH64_AARCH64SUBTARGET_H
 #define LLVM_LIB_TARGET_AARCH64_AARCH64SUBTARGET_H
 
 #include "AArch64FrameLowering.h"
 #include "AArch64ISelLowering.h"
 #include "AArch64InstrInfo.h"
 #include "AArch64RegisterInfo.h"
 #include "AArch64SelectionDAGInfo.h"
 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
 #include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/DataLayout.h"
 #include <string>
 
 #define GET_SUBTARGETINFO_HEADER
 #include "AArch64GenSubtargetInfo.inc"
 
 namespace llvm {
 class GlobalValue;
 class StringRef;
 class Triple;
 
 class AArch64Subtarget final : public AArch64GenSubtargetInfo {
 public:
   enum ARMProcFamilyEnum : uint8_t {
     Others,
     A64FX,
+    Ampere1,
     AppleA7,
     AppleA10,
     AppleA11,
     AppleA12,
     AppleA13,
     AppleA14,
     Carmel,
     CortexA35,
     CortexA53,
     CortexA55,
     CortexA510,
     CortexA57,
     CortexA65,
     CortexA72,
     CortexA73,
     CortexA75,
     CortexA76,
     CortexA77,
     CortexA78,
     CortexA78C,
     CortexA710,
     CortexR82,
     CortexX1,
     CortexX1C,
     CortexX2,
     ExynosM3,
     Falkor,
     Kryo,
     NeoverseE1,
     NeoverseN1,
     NeoverseN2,
     Neoverse512TVB,
     NeoverseV1,
     Saphira,
     ThunderX2T99,
     ThunderX,
     ThunderXT81,
     ThunderXT83,
     ThunderXT88,
     ThunderX3T110,
     TSV110
   };
 
 protected:
   /// ARMProcFamily - ARM processor family: Cortex-A53, Cortex-A57, and others.
   ARMProcFamilyEnum ARMProcFamily = Others;
 
   bool HasV8_0aOps = false;
   bool HasV8_1aOps = false;
   bool HasV8_2aOps = false;
   bool HasV8_3aOps = false;
   bool HasV8_4aOps = false;
   bool HasV8_5aOps = false;
   bool HasV8_6aOps = false;
   bool HasV8_7aOps = false;
   bool HasV8_8aOps = false;
   bool HasV9_0aOps = false;
   bool HasV9_1aOps = false;
   bool HasV9_2aOps = false;
   bool HasV9_3aOps = false;
   bool HasV8_0rOps = false;
 
   bool HasCONTEXTIDREL2 = false;
   bool HasEL2VMSA = false;
   bool HasEL3 = false;
   bool HasFPARMv8 = false;
   bool HasNEON = false;
   bool HasCrypto = false;
   bool HasDotProd = false;
   bool HasCRC = false;
   bool HasLSE = false;
   bool HasLSE2 = false;
   bool HasRAS = false;
   bool HasRDM = false;
   bool HasPerfMon = false;
   bool HasFullFP16 = false;
   bool HasFP16FML = false;
   bool HasSPE = false;
 
   bool FixCortexA53_835769 = false;
 
   // ARMv8.1 extensions
   bool HasVH = false;
   bool HasPAN = false;
   bool HasLOR = false;
 
   // ARMv8.2 extensions
   bool HasPsUAO = false;
   bool HasPAN_RWV = false;
   bool HasCCPP = false;
 
   // SVE extensions
   bool HasSVE = false;
   bool UseExperimentalZeroingPseudos = false;
   bool UseScalarIncVL = false;
 
   // Armv8.2 Crypto extensions
   bool HasSM4 = false;
   bool HasSHA3 = false;
   bool HasSHA2 = false;
   bool HasAES = false;
 
   // ARMv8.3 extensions
   bool HasPAuth = false;
   bool HasJS = false;
   bool HasCCIDX = false;
   bool HasComplxNum = false;
 
   // ARMv8.4 extensions
   bool HasNV = false;
   bool HasMPAM = false;
   bool HasDIT = false;
   bool HasTRACEV8_4 = false;
   bool HasAM = false;
   bool HasSEL2 = false;
   bool HasTLB_RMI = false;
   bool HasFlagM = false;
   bool HasRCPC_IMMO = false;
 
   bool HasLSLFast = false;
   bool HasRCPC = false;
   bool HasAggressiveFMA = false;
 
   // Armv8.5-A Extensions
   bool HasAlternativeNZCV = false;
   bool HasFRInt3264 = false;
   bool HasSpecRestrict = false;
   bool HasSSBS = false;
   bool HasSB = false;
   bool HasPredRes = false;
   bool HasCCDP = false;
   bool HasBTI = false;
   bool HasRandGen = false;
   bool HasMTE = false;
   bool HasTME = false;
 
   // Armv8.6-A Extensions
   bool HasBF16 = false;
   bool HasMatMulInt8 = false;
   bool HasMatMulFP32 = false;
   bool HasMatMulFP64 = false;
   bool HasAMVS = false;
   bool HasFineGrainedTraps = false;
   bool HasEnhancedCounterVirtualization = false;
 
   // Armv8.7-A Extensions
   bool HasXS = false;
   bool HasWFxT = false;
   bool HasHCX = false;
   bool HasLS64 = false;
 
   // Armv8.8-A Extensions
   bool HasHBC = false;
   bool HasMOPS = false;
 
   // Arm SVE2 extensions
   bool HasSVE2 = false;
   bool HasSVE2AES = false;
   bool HasSVE2SM4 = false;
   bool HasSVE2SHA3 = false;
   bool HasSVE2BitPerm = false;
 
   // Armv9-A Extensions
   bool HasRME = false;
 
   // Arm Scalable Matrix Extension (SME)
   bool HasSME = false;
   bool HasSMEF64 = false;
   bool HasSMEI64 = false;
   bool HasStreamingSVE = false;
 
   // AppleA7 system register.
   bool HasAppleA7SysReg = false;
 
   // Future architecture extensions.
   bool HasETE = false;
   bool HasTRBE = false;
   bool HasBRBE = false;
   bool HasSPE_EEF = false;
 
   // HasZeroCycleRegMove - Has zero-cycle register mov instructions.
   bool HasZeroCycleRegMove = false;
 
   // HasZeroCycleZeroing - Has zero-cycle zeroing instructions.
   bool HasZeroCycleZeroing = false;
   bool HasZeroCycleZeroingGP = false;
   bool HasZeroCycleZeroingFPWorkaround = false;
 
   // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0".
   // as movi is more efficient across all cores. Newer cores can eliminate
   // fmovs early and there is no difference with movi, but this not true for
   // all implementations.
   bool HasZeroCycleZeroingFP = true;
 
   // StrictAlign - Disallow unaligned memory accesses.
   bool StrictAlign = false;
 
   // NegativeImmediates - transform instructions with negative immediates
   bool NegativeImmediates = true;
 
   // Enable 64-bit vectorization in SLP.
   unsigned MinVectorRegisterBitWidth = 64;
 
   // Do not place a BTI instruction after a call to a return twice function like
   // setjmp.
   bool NoBTIAtReturnTwice = false;
 
   bool OutlineAtomics = false;
   bool PredictableSelectIsExpensive = false;
   bool BalanceFPOps = false;
   bool CustomAsCheapAsMove = false;
   bool ExynosAsCheapAsMove = false;
   bool UsePostRAScheduler = false;
   bool Misaligned128StoreIsSlow = false;
   bool Paired128IsSlow = false;
   bool STRQroIsSlow = false;
   bool UseAlternateSExtLoadCVTF32Pattern = false;
   bool HasArithmeticBccFusion = false;
   bool HasArithmeticCbzFusion = false;
   bool HasCmpBccFusion = false;
   bool HasFuseAddress = false;
   bool HasFuseAES = false;
   bool HasFuseArithmeticLogic = false;
   bool HasFuseCCSelect = false;
   bool HasFuseCryptoEOR = false;
   bool HasFuseLiterals = false;
   bool DisableLatencySchedHeuristic = false;
   bool UseRSqrt = false;
   bool Force32BitJumpTables = false;
   bool UseEL1ForTP = false;
   bool UseEL2ForTP = false;
   bool UseEL3ForTP = false;
   bool AllowTaggedGlobals = false;
   bool HardenSlsRetBr = false;
   bool HardenSlsBlr = false;
   bool HardenSlsNoComdat = false;
   uint8_t MaxInterleaveFactor = 2;
   uint8_t VectorInsertExtractBaseCost = 3;
   uint16_t CacheLineSize = 0;
   uint16_t PrefetchDistance = 0;
   uint16_t MinPrefetchStride = 1;
   unsigned MaxPrefetchIterationsAhead = UINT_MAX;
   unsigned PrefFunctionLogAlignment = 0;
   unsigned PrefLoopLogAlignment = 0;
   unsigned MaxBytesForLoopAlignment = 0;
   unsigned MaxJumpTableSize = 0;
   unsigned WideningBaseCost = 0;
 
   // ReserveXRegister[i] - X#i is not available as a general purpose register.
   BitVector ReserveXRegister;
 
   // CustomCallUsedXRegister[i] - X#i call saved.
   BitVector CustomCallSavedXRegs;
 
   bool IsLittle;
 
   unsigned MinSVEVectorSizeInBits;
   unsigned MaxSVEVectorSizeInBits;
   unsigned VScaleForTuning = 2;
 
   /// TargetTriple - What processor and OS we're targeting.
   Triple TargetTriple;
 
   AArch64FrameLowering FrameLowering;
   AArch64InstrInfo InstrInfo;
   AArch64SelectionDAGInfo TSInfo;
   AArch64TargetLowering TLInfo;
 
   /// GlobalISel related APIs.
   std::unique_ptr<CallLowering> CallLoweringInfo;
   std::unique_ptr<InlineAsmLowering> InlineAsmLoweringInfo;
   std::unique_ptr<InstructionSelector> InstSelector;
   std::unique_ptr<LegalizerInfo> Legalizer;
   std::unique_ptr<RegisterBankInfo> RegBankInfo;
 
 private:
   /// initializeSubtargetDependencies - Initializes using CPUString and the
   /// passed in feature string so that we can use initializer lists for
   /// subtarget initialization.
   AArch64Subtarget &initializeSubtargetDependencies(StringRef FS,
                                                     StringRef CPUString,
                                                     StringRef TuneCPUString);
 
   /// Initialize properties based on the selected processor family.
   void initializeProperties();
 
 public:
   /// This constructor initializes the data members to match that
   /// of the specified triple.
   AArch64Subtarget(const Triple &TT, const std::string &CPU,
                    const std::string &TuneCPU, const std::string &FS,
                    const TargetMachine &TM, bool LittleEndian,
                    unsigned MinSVEVectorSizeInBitsOverride = 0,
                    unsigned MaxSVEVectorSizeInBitsOverride = 0);
 
   const AArch64SelectionDAGInfo *getSelectionDAGInfo() const override {
     return &TSInfo;
   }
   const AArch64FrameLowering *getFrameLowering() const override {
     return &FrameLowering;
   }
   const AArch64TargetLowering *getTargetLowering() const override {
     return &TLInfo;
   }
   const AArch64InstrInfo *getInstrInfo() const override { return &InstrInfo; }
   const AArch64RegisterInfo *getRegisterInfo() const override {
     return &getInstrInfo()->getRegisterInfo();
   }
   const CallLowering *getCallLowering() const override;
   const InlineAsmLowering *getInlineAsmLowering() const override;
   InstructionSelector *getInstructionSelector() const override;
   const LegalizerInfo *getLegalizerInfo() const override;
   const RegisterBankInfo *getRegBankInfo() const override;
   const Triple &getTargetTriple() const { return TargetTriple; }
   bool enableMachineScheduler() const override { return true; }
   bool enablePostRAScheduler() const override {
     return UsePostRAScheduler;
   }
 
   /// Returns ARM processor family.
   /// Avoid this function! CPU specifics should be kept local to this class
   /// and preferably modeled with SubtargetFeatures or properties in
   /// initializeProperties().
   ARMProcFamilyEnum getProcFamily() const {
     return ARMProcFamily;
   }
 
   bool hasV8_0aOps() const { return HasV8_0aOps; }
   bool hasV8_1aOps() const { return HasV8_1aOps; }
   bool hasV8_2aOps() const { return HasV8_2aOps; }
   bool hasV8_3aOps() const { return HasV8_3aOps; }
   bool hasV8_4aOps() const { return HasV8_4aOps; }
   bool hasV8_5aOps() const { return HasV8_5aOps; }
   bool hasV9_0aOps() const { return HasV9_0aOps; }
   bool hasV9_1aOps() const { return HasV9_1aOps; }
   bool hasV9_2aOps() const { return HasV9_2aOps; }
   bool hasV9_3aOps() const { return HasV9_3aOps; }
   bool hasV8_0rOps() const { return HasV8_0rOps; }
 
   bool hasZeroCycleRegMove() const { return HasZeroCycleRegMove; }
 
   bool hasZeroCycleZeroingGP() const { return HasZeroCycleZeroingGP; }
 
   bool hasZeroCycleZeroingFP() const { return HasZeroCycleZeroingFP; }
 
   bool hasZeroCycleZeroingFPWorkaround() const {
     return HasZeroCycleZeroingFPWorkaround;
   }
 
   bool requiresStrictAlign() const { return StrictAlign; }
 
   bool isXRaySupported() const override { return true; }
 
   unsigned getMinVectorRegisterBitWidth() const {
     return MinVectorRegisterBitWidth;
   }
 
   bool isXRegisterReserved(size_t i) const { return ReserveXRegister[i]; }
   unsigned getNumXRegisterReserved() const { return ReserveXRegister.count(); }
   bool isXRegCustomCalleeSaved(size_t i) const {
     return CustomCallSavedXRegs[i];
   }
   bool hasCustomCallingConv() const { return CustomCallSavedXRegs.any(); }
   bool hasFPARMv8() const { return HasFPARMv8; }
   bool hasNEON() const { return HasNEON; }
   bool hasCrypto() const { return HasCrypto; }
   bool hasDotProd() const { return HasDotProd; }
   bool hasCRC() const { return HasCRC; }
   bool hasLSE() const { return HasLSE; }
   bool hasLSE2() const { return HasLSE2; }
   bool hasRAS() const { return HasRAS; }
   bool hasRDM() const { return HasRDM; }
   bool hasSM4() const { return HasSM4; }
   bool hasSHA3() const { return HasSHA3; }
   bool hasSHA2() const { return HasSHA2; }
   bool hasAES() const { return HasAES; }
   bool hasCONTEXTIDREL2() const { return HasCONTEXTIDREL2; }
   bool balanceFPOps() const { return BalanceFPOps; }
   bool predictableSelectIsExpensive() const {
     return PredictableSelectIsExpensive;
   }
   bool hasCustomCheapAsMoveHandling() const { return CustomAsCheapAsMove; }
   bool hasExynosCheapAsMoveHandling() const { return ExynosAsCheapAsMove; }
   bool isMisaligned128StoreSlow() const { return Misaligned128StoreIsSlow; }
   bool isPaired128Slow() const { return Paired128IsSlow; }
   bool isSTRQroSlow() const { return STRQroIsSlow; }
   bool useAlternateSExtLoadCVTF32Pattern() const {
     return UseAlternateSExtLoadCVTF32Pattern;
   }
   bool hasArithmeticBccFusion() const { return HasArithmeticBccFusion; }
   bool hasArithmeticCbzFusion() const { return HasArithmeticCbzFusion; }
   bool hasCmpBccFusion() const { return HasCmpBccFusion; }
   bool hasFuseAddress() const { return HasFuseAddress; }
   bool hasFuseAES() const { return HasFuseAES; }
   bool hasFuseArithmeticLogic() const { return HasFuseArithmeticLogic; }
   bool hasFuseCCSelect() const { return HasFuseCCSelect; }
   bool hasFuseCryptoEOR() const { return HasFuseCryptoEOR; }
   bool hasFuseLiterals() const { return HasFuseLiterals; }
 
   /// Return true if the CPU supports any kind of instruction fusion.
   bool hasFusion() const {
     return hasArithmeticBccFusion() || hasArithmeticCbzFusion() ||
            hasFuseAES() || hasFuseArithmeticLogic() ||
            hasFuseCCSelect() || hasFuseLiterals();
   }
 
   bool hardenSlsRetBr() const { return HardenSlsRetBr; }
   bool hardenSlsBlr() const { return HardenSlsBlr; }
   bool hardenSlsNoComdat() const { return HardenSlsNoComdat; }
 
   bool useEL1ForTP() const { return UseEL1ForTP; }
   bool useEL2ForTP() const { return UseEL2ForTP; }
   bool useEL3ForTP() const { return UseEL3ForTP; }
 
   bool useRSqrt() const { return UseRSqrt; }
   bool force32BitJumpTables() const { return Force32BitJumpTables; }
   unsigned getMaxInterleaveFactor() const { return MaxInterleaveFactor; }
   unsigned getVectorInsertExtractBaseCost() const {
     return VectorInsertExtractBaseCost;
   }
   unsigned getCacheLineSize() const override { return CacheLineSize; }
   unsigned getPrefetchDistance() const override { return PrefetchDistance; }
   unsigned getMinPrefetchStride(unsigned NumMemAccesses,
                                 unsigned NumStridedMemAccesses,
                                 unsigned NumPrefetches,
                                 bool HasCall) const override {
     return MinPrefetchStride;
   }
   unsigned getMaxPrefetchIterationsAhead() const override {
     return MaxPrefetchIterationsAhead;
   }
   unsigned getPrefFunctionLogAlignment() const {
     return PrefFunctionLogAlignment;
   }
   unsigned getPrefLoopLogAlignment() const { return PrefLoopLogAlignment; }
 
   unsigned getMaxBytesForLoopAlignment() const {
     return MaxBytesForLoopAlignment;
   }
 
   unsigned getMaximumJumpTableSize() const { return MaxJumpTableSize; }
 
   unsigned getWideningBaseCost() const { return WideningBaseCost; }
 
   bool useExperimentalZeroingPseudos() const {
     return UseExperimentalZeroingPseudos;
   }
 
   bool useScalarIncVL() const { return UseScalarIncVL; }
 
   /// CPU has TBI (top byte of addresses is ignored during HW address
   /// translation) and OS enables it.
   bool supportsAddressTopByteIgnored() const;
 
   bool hasPerfMon() const { return HasPerfMon; }
   bool hasFullFP16() const { return HasFullFP16; }
   bool hasFP16FML() const { return HasFP16FML; }
   bool hasSPE() const { return HasSPE; }
   bool hasLSLFast() const { return HasLSLFast; }
   bool hasSVE() const { return HasSVE; }
   bool hasSVE2() const { return HasSVE2; }
   bool hasRCPC() const { return HasRCPC; }
   bool hasAggressiveFMA() const { return HasAggressiveFMA; }
   bool hasAlternativeNZCV() const { return HasAlternativeNZCV; }
   bool hasFRInt3264() const { return HasFRInt3264; }
   bool hasSpecRestrict() const { return HasSpecRestrict; }
   bool hasSSBS() const { return HasSSBS; }
   bool hasSB() const { return HasSB; }
   bool hasPredRes() const { return HasPredRes; }
   bool hasCCDP() const { return HasCCDP; }
   bool hasBTI() const { return HasBTI; }
   bool hasRandGen() const { return HasRandGen; }
   bool hasMTE() const { return HasMTE; }
   bool hasTME() const { return HasTME; }
   // Arm SVE2 extensions
   bool hasSVE2AES() const { return HasSVE2AES; }
   bool hasSVE2SM4() const { return HasSVE2SM4; }
   bool hasSVE2SHA3() const { return HasSVE2SHA3; }
   bool hasSVE2BitPerm() const { return HasSVE2BitPerm; }
   bool hasMatMulInt8() const { return HasMatMulInt8; }
   bool hasMatMulFP32() const { return HasMatMulFP32; }
   bool hasMatMulFP64() const { return HasMatMulFP64; }
 
   // Armv8.6-A Extensions
   bool hasBF16() const { return HasBF16; }
   bool hasFineGrainedTraps() const { return HasFineGrainedTraps; }
   bool hasEnhancedCounterVirtualization() const {
     return HasEnhancedCounterVirtualization;
   }
 
   // Arm Scalable Matrix Extension (SME)
   bool hasSME() const { return HasSME; }
   bool hasSMEF64() const { return HasSMEF64; }
   bool hasSMEI64() const { return HasSMEI64; }
   bool hasStreamingSVE() const { return HasStreamingSVE; }
 
   bool isLittleEndian() const { return IsLittle; }
 
   bool isTargetDarwin() const { return TargetTriple.isOSDarwin(); }
   bool isTargetIOS() const { return TargetTriple.isiOS(); }
   bool isTargetLinux() const { return TargetTriple.isOSLinux(); }
   bool isTargetWindows() const { return TargetTriple.isOSWindows(); }
   bool isTargetAndroid() const { return TargetTriple.isAndroid(); }
   bool isTargetFuchsia() const { return TargetTriple.isOSFuchsia(); }
 
   bool isTargetCOFF() const { return TargetTriple.isOSBinFormatCOFF(); }
   bool isTargetELF() const { return TargetTriple.isOSBinFormatELF(); }
   bool isTargetMachO() const { return TargetTriple.isOSBinFormatMachO(); }
 
   bool isTargetILP32() const {
     return TargetTriple.isArch32Bit() ||
            TargetTriple.getEnvironment() == Triple::GNUILP32;
   }
 
   bool useAA() const override;
 
   bool outlineAtomics() const { return OutlineAtomics; }
 
   bool hasVH() const { return HasVH; }
   bool hasPAN() const { return HasPAN; }
   bool hasLOR() const { return HasLOR; }
 
   bool hasPsUAO() const { return HasPsUAO; }
   bool hasPAN_RWV() const { return HasPAN_RWV; }
   bool hasCCPP() const { return HasCCPP; }
 
   bool hasPAuth() const { return HasPAuth; }
   bool hasJS() const { return HasJS; }
   bool hasCCIDX() const { return HasCCIDX; }
   bool hasComplxNum() const { return HasComplxNum; }
 
   bool hasNV() const { return HasNV; }
   bool hasMPAM() const { return HasMPAM; }
   bool hasDIT() const { return HasDIT; }
   bool hasTRACEV8_4() const { return HasTRACEV8_4; }
   bool hasAM() const { return HasAM; }
   bool hasAMVS() const { return HasAMVS; }
   bool hasXS() const { return HasXS; }
   bool hasWFxT() const { return HasWFxT; }
   bool hasHCX() const { return HasHCX; }
   bool hasLS64() const { return HasLS64; }
   bool hasSEL2() const { return HasSEL2; }
   bool hasTLB_RMI() const { return HasTLB_RMI; }
   bool hasFlagM() const { return HasFlagM; }
   bool hasRCPC_IMMO() const { return HasRCPC_IMMO; }
   bool hasEL2VMSA() const { return HasEL2VMSA; }
   bool hasEL3() const { return HasEL3; }
   bool hasHBC() const { return HasHBC; }
   bool hasMOPS() const { return HasMOPS; }
 
   bool fixCortexA53_835769() const { return FixCortexA53_835769; }
 
   bool noBTIAtReturnTwice() const { return NoBTIAtReturnTwice; }
 
   bool addrSinkUsingGEPs() const override {
     // Keeping GEPs inbounds is important for exploiting AArch64
     // addressing-modes in ILP32 mode.
     return useAA() || isTargetILP32();
   }
 
   bool useSmallAddressing() const {
     switch (TLInfo.getTargetMachine().getCodeModel()) {
       case CodeModel::Kernel:
         // Kernel is currently allowed only for Fuchsia targets,
         // where it is the same as Small for almost all purposes.
       case CodeModel::Small:
         return true;
       default:
         return false;
     }
   }
 
   /// ParseSubtargetFeatures - Parses features string setting specified
   /// subtarget options.  Definition of function is auto generated by tblgen.
   void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS);
 
   /// ClassifyGlobalReference - Find the target operand flags that describe
   /// how a global value should be referenced for the current subtarget.
   unsigned ClassifyGlobalReference(const GlobalValue *GV,
                                    const TargetMachine &TM) const;
 
   unsigned classifyGlobalFunctionReference(const GlobalValue *GV,
                                            const TargetMachine &TM) const;
 
   void overrideSchedPolicy(MachineSchedPolicy &Policy,
                            unsigned NumRegionInstrs) const override;
 
   bool enableEarlyIfConversion() const override;
 
   bool enableAdvancedRASplitCost() const override { return false; }
 
   std::unique_ptr<PBQPRAConstraint> getCustomPBQPConstraints() const override;
 
   bool isCallingConvWin64(CallingConv::ID CC) const {
     switch (CC) {
     case CallingConv::C:
     case CallingConv::Fast:
     case CallingConv::Swift:
       return isTargetWindows();
     case CallingConv::Win64:
       return true;
     default:
       return false;
     }
   }
 
   /// Return whether FrameLowering should always set the "extended frame
   /// present" bit in FP, or set it based on a symbol in the runtime.
   bool swiftAsyncContextIsDynamicallySet() const {
     // Older OS versions (particularly system unwinders) are confused by the
     // Swift extended frame, so when building code that might be run on them we
     // must dynamically query the concurrency library to determine whether
     // extended frames should be flagged as present.
     const Triple &TT = getTargetTriple();
 
     unsigned Major = TT.getOSVersion().getMajor();
     switch(TT.getOS()) {
     default:
       return false;
     case Triple::IOS:
     case Triple::TvOS:
       return Major < 15;
     case Triple::WatchOS:
       return Major < 8;
     case Triple::MacOSX:
     case Triple::Darwin:
       return Major < 12;
     }
   }
 
   void mirFileLoaded(MachineFunction &MF) const override;
 
   // Return the known range for the bit length of SVE data registers. A value
   // of 0 means nothing is known about that particular limit beyong what's
   // implied by the architecture.
   unsigned getMaxSVEVectorSizeInBits() const {
     assert(HasSVE && "Tried to get SVE vector length without SVE support!");
     return MaxSVEVectorSizeInBits;
   }
 
   unsigned getMinSVEVectorSizeInBits() const {
     assert(HasSVE && "Tried to get SVE vector length without SVE support!");
     return MinSVEVectorSizeInBits;
   }
 
   bool useSVEForFixedLengthVectors() const {
     // Prefer NEON unless larger SVE registers are available.
     return hasSVE() && getMinSVEVectorSizeInBits() >= 256;
   }
 
   unsigned getVScaleForTuning() const { return VScaleForTuning; }
 };
 } // End llvm namespace
 
 #endif
diff --git a/contrib/llvm-project/llvm/lib/Target/AVR/AVRCallingConv.td b/contrib/llvm-project/llvm/lib/Target/AVR/AVRCallingConv.td
index b4bc35e191c0..0fae61fb55c5 100644
--- a/contrib/llvm-project/llvm/lib/Target/AVR/AVRCallingConv.td
+++ b/contrib/llvm-project/llvm/lib/Target/AVR/AVRCallingConv.td
@@ -1,39 +1,41 @@
 //===-- AVRCallingConv.td - Calling Conventions for AVR ----*- tablegen -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This describes the calling conventions for AVR architecture.
 // Normal functions use a special calling convention, solved in code.
 //===----------------------------------------------------------------------===//
 
 //===----------------------------------------------------------------------===//
 // AVR Return Value Calling Convention
 //===----------------------------------------------------------------------===//
 
 // Special return value calling convention for runtime functions.
 def RetCC_AVR_BUILTIN : CallingConv<[
   CCIfType<[i8], CCAssignToReg<[R24, R25]>>,
   CCIfType<[i16], CCAssignToReg<[R23R22, R25R24]>>
 ]>;
 
 //===----------------------------------------------------------------------===//
 // AVR Argument Calling Conventions
 //===----------------------------------------------------------------------===//
 
 // The calling conventions are implemented in custom C++ code
 
 // Calling convention for variadic functions.
 def ArgCC_AVR_Vararg : CallingConv<[
+  // i8 are always passed through the stack with a byte slot and byte alignment.
+  CCIfType<[i8], CCAssignToStack<1, 1>>,
   // i16 are always passed through the stack with an alignment of 1.
   CCAssignToStack<2, 1>
 ]>;
 
 //===----------------------------------------------------------------------===//
 // Callee-saved register lists.
 //===----------------------------------------------------------------------===//
 
 def CSR_Normal : CalleeSavedRegs<(add R29, R28, (sequence "R%u", 17, 2))>;
 def CSR_Interrupts : CalleeSavedRegs<(add(sequence "R%u", 31, 2))>;
diff --git a/contrib/llvm-project/llvm/lib/Target/AVR/AVRInstrInfo.td b/contrib/llvm-project/llvm/lib/Target/AVR/AVRInstrInfo.td
index 2b96dc0b833a..7e027369f096 100644
--- a/contrib/llvm-project/llvm/lib/Target/AVR/AVRInstrInfo.td
+++ b/contrib/llvm-project/llvm/lib/Target/AVR/AVRInstrInfo.td
@@ -1,2528 +1,2532 @@
 //===-- AVRInstrInfo.td - AVR Instruction defs -------------*- tablegen -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file describes the AVR instructions in TableGen format.
 //
 //===----------------------------------------------------------------------===//
 
 include "AVRInstrFormats.td"
 
 //===----------------------------------------------------------------------===//
 // AVR Type Profiles
 //===----------------------------------------------------------------------===//
 
 def SDT_AVRCallSeqStart : SDCallSeqStart<[SDTCisVT<0, i16>, SDTCisVT<1, i16>]>;
 def SDT_AVRCallSeqEnd : SDCallSeqEnd<[SDTCisVT<0, i16>, SDTCisVT<1, i16>]>;
 def SDT_AVRCall : SDTypeProfile<0, -1, [SDTCisVT<0, iPTR>]>;
 def SDT_AVRWrapper : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>, SDTCisPtrTy<0>]>;
 def SDT_AVRBrcond
     : SDTypeProfile<0, 2, [SDTCisVT<0, OtherVT>, SDTCisVT<1, i8>]>;
 def SDT_AVRCmp : SDTypeProfile<0, 2, [SDTCisSameAs<0, 1>]>;
 def SDT_AVRTst : SDTypeProfile<0, 1, [SDTCisInt<0>]>;
 def SDT_AVRSelectCC
     : SDTypeProfile<1, 3,
                     [SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>, SDTCisVT<3, i8>]>;
 
 //===----------------------------------------------------------------------===//
 // AVR Specific Node Definitions
 //===----------------------------------------------------------------------===//
 
 def AVRretflag : SDNode<"AVRISD::RET_FLAG", SDTNone,
                         [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
 def AVRretiflag : SDNode<"AVRISD::RETI_FLAG", SDTNone,
                          [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
 
 def AVRcallseq_start : SDNode<"ISD::CALLSEQ_START", SDT_AVRCallSeqStart,
                               [SDNPHasChain, SDNPOutGlue]>;
 def AVRcallseq_end : SDNode<"ISD::CALLSEQ_END", SDT_AVRCallSeqEnd,
                             [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
 
 def AVRcall : SDNode<"AVRISD::CALL", SDT_AVRCall,
                      [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue, SDNPVariadic]>;
 
 def AVRWrapper : SDNode<"AVRISD::WRAPPER", SDT_AVRWrapper>;
 
 def AVRbrcond
     : SDNode<"AVRISD::BRCOND", SDT_AVRBrcond, [SDNPHasChain, SDNPInGlue]>;
 def AVRcmp : SDNode<"AVRISD::CMP", SDT_AVRCmp, [SDNPOutGlue]>;
 def AVRcmpc : SDNode<"AVRISD::CMPC", SDT_AVRCmp, [SDNPInGlue, SDNPOutGlue]>;
 def AVRtst : SDNode<"AVRISD::TST", SDT_AVRTst, [SDNPOutGlue]>;
 def AVRselectcc : SDNode<"AVRISD::SELECT_CC", SDT_AVRSelectCC, [SDNPInGlue]>;
 
 // Shift nodes.
 def AVRlsl : SDNode<"AVRISD::LSL", SDTIntUnaryOp>;
 def AVRlsr : SDNode<"AVRISD::LSR", SDTIntUnaryOp>;
 def AVRrol : SDNode<"AVRISD::ROL", SDTIntUnaryOp>;
 def AVRror : SDNode<"AVRISD::ROR", SDTIntUnaryOp>;
 def AVRasr : SDNode<"AVRISD::ASR", SDTIntUnaryOp>;
 def AVRlslhi : SDNode<"AVRISD::LSLHI", SDTIntUnaryOp>;
 def AVRlsrlo : SDNode<"AVRISD::LSRLO", SDTIntUnaryOp>;
 def AVRasrlo : SDNode<"AVRISD::ASRLO", SDTIntUnaryOp>;
 def AVRlslbn : SDNode<"AVRISD::LSLBN", SDTIntBinOp>;
 def AVRlsrbn : SDNode<"AVRISD::LSRBN", SDTIntBinOp>;
 def AVRasrbn : SDNode<"AVRISD::ASRBN", SDTIntBinOp>;
 def AVRlslwn : SDNode<"AVRISD::LSLWN", SDTIntBinOp>;
 def AVRlsrwn : SDNode<"AVRISD::LSRWN", SDTIntBinOp>;
 def AVRasrwn : SDNode<"AVRISD::ASRWN", SDTIntBinOp>;
 
 // Pseudo shift nodes for non-constant shift amounts.
 def AVRlslLoop : SDNode<"AVRISD::LSLLOOP", SDTIntShiftOp>;
 def AVRlsrLoop : SDNode<"AVRISD::LSRLOOP", SDTIntShiftOp>;
 def AVRrolLoop : SDNode<"AVRISD::ROLLOOP", SDTIntShiftOp>;
 def AVRrorLoop : SDNode<"AVRISD::RORLOOP", SDTIntShiftOp>;
 def AVRasrLoop : SDNode<"AVRISD::ASRLOOP", SDTIntShiftOp>;
 
 // SWAP node.
 def AVRSwap : SDNode<"AVRISD::SWAP", SDTIntUnaryOp>;
 
 //===----------------------------------------------------------------------===//
 // AVR Operands, Complex Patterns and Transformations Definitions.
 //===----------------------------------------------------------------------===//
 
 def imm8_neg_XFORM : SDNodeXForm<imm, [{
                                    return CurDAG->getTargetConstant(
                                        -N->getAPIntValue(), SDLoc(N), MVT::i8);
                                  }]>;
 
 def imm16_neg_XFORM
     : SDNodeXForm<imm, [{
                     return CurDAG->getTargetConstant(-N->getAPIntValue(),
                                                      SDLoc(N), MVT::i16);
                   }]>;
 
 def imm0_63_neg : PatLeaf<(imm), [{
                             int64_t val = -N->getSExtValue();
                             return val >= 0 && val < 64;
                           }],
                           imm16_neg_XFORM>;
 
 def uimm6 : PatLeaf<(imm), [{ return isUInt<6>(N->getZExtValue()); }]>;
 
 // imm_com8_XFORM - Return the complement of a imm_com8 value
 def imm_com8_XFORM
     : SDNodeXForm<imm, [{
                     return CurDAG->getTargetConstant(
                         ~((uint8_t) N->getZExtValue()), SDLoc(N), MVT::i8);
                   }]>;
 
 // imm_com8 - Match an immediate that is a complement
 // of a 8-bit immediate.
 // Note: this pattern doesn't require an encoder method and such, as it's
 // only used on aliases (Pat<> and InstAlias<>). The actual encoding
 // is handled by the destination instructions, which use imm_com8.
 def imm_com8_asmoperand : AsmOperandClass { let Name = "ImmCom8"; }
 def imm_com8 : Operand<i8> { let ParserMatchClass = imm_com8_asmoperand; }
 
 def ioaddr_XFORM
     : SDNodeXForm<imm, [{
                     uint8_t offset = Subtarget->getIORegisterOffset();
                     return CurDAG->getTargetConstant(
                         uint8_t(N->getZExtValue()) - offset, SDLoc(N), MVT::i8);
                   }]>;
 
 def iobitpos8_XFORM
     : SDNodeXForm<imm, [{
                     return CurDAG->getTargetConstant(
                         Log2_32(uint8_t(N->getZExtValue())), SDLoc(N), MVT::i8);
                   }]>;
 
 def iobitposn8_XFORM : SDNodeXForm<imm, [{
                                      return CurDAG->getTargetConstant(
                                          Log2_32(uint8_t(~N->getZExtValue())),
                                          SDLoc(N), MVT::i8);
                                    }]>;
 
 def ioaddr8 : PatLeaf<(imm), [{
                         uint8_t offset = Subtarget->getIORegisterOffset();
                         uint64_t val = N->getZExtValue() - offset;
                         return val < 0x40;
                       }],
                       ioaddr_XFORM>;
 
 def lowioaddr8 : PatLeaf<(imm), [{
                            uint8_t offset = Subtarget->getIORegisterOffset();
                            uint64_t val = N->getZExtValue() - offset;
                            return val < 0x20;
                          }],
                          ioaddr_XFORM>;
 
 def ioaddr16 : PatLeaf<(imm), [{
                          uint8_t offset = Subtarget->getIORegisterOffset();
                          uint64_t val = N->getZExtValue() - offset;
                          return val < 0x3f;
                        }],
                        ioaddr_XFORM>;
 
 def iobitpos8
     : PatLeaf<(imm), [{ return isPowerOf2_32(uint8_t(N->getZExtValue())); }],
               iobitpos8_XFORM>;
 
 def iobitposn8
     : PatLeaf<(imm), [{ return isPowerOf2_32(uint8_t(~N->getZExtValue())); }],
               iobitposn8_XFORM>;
 
 def MemriAsmOperand : AsmOperandClass {
   let Name = "Memri";
   let ParserMethod = "parseMemriOperand";
 }
 
 /// Address operand for `reg+imm` used by STD and LDD.
 def memri : Operand<iPTR> {
   let MIOperandInfo = (ops PTRDISPREGS, i16imm);
 
   let PrintMethod = "printMemri";
   let EncoderMethod = "encodeMemri";
 
   let ParserMatchClass = MemriAsmOperand;
 }
 
 // Address operand for `SP+imm` used by STD{W}SPQRr
 def memspi : Operand<iPTR> { let MIOperandInfo = (ops GPRSP, i16imm); }
 
 def relbrtarget_7 : Operand<OtherVT> {
   let PrintMethod = "printPCRelImm";
   let EncoderMethod = "encodeRelCondBrTarget<AVR::fixup_7_pcrel>";
 }
 
 def brtarget_13 : Operand<OtherVT> {
   let PrintMethod = "printPCRelImm";
   let EncoderMethod = "encodeRelCondBrTarget<AVR::fixup_13_pcrel>";
 }
 
+def rcalltarget_13 : Operand<i16> {
+  let PrintMethod = "printPCRelImm";
+  let EncoderMethod = "encodeRelCondBrTarget<AVR::fixup_13_pcrel>";
+}
+
 // The target of a 22 or 16-bit call/jmp instruction.
 def call_target : Operand<iPTR> {
   let EncoderMethod = "encodeCallTarget";
   let DecoderMethod = "decodeCallTarget";
 }
 
 // A 16-bit address (which can lead to an R_AVR_16 relocation).
 def imm16 : Operand<i16> { let EncoderMethod = "encodeImm<AVR::fixup_16, 2>"; }
 
 /// A 6-bit immediate used in the ADIW/SBIW instructions.
 def imm_arith6 : Operand<i16> {
   let EncoderMethod = "encodeImm<AVR::fixup_6_adiw, 0>";
 }
 
 /// An 8-bit immediate inside an instruction with the same format
 /// as the `LDI` instruction (the `FRdK` format).
 def imm_ldi8 : Operand<i8> {
   let EncoderMethod = "encodeImm<AVR::fixup_ldi, 0>";
 }
 
 /// A 5-bit port number used in SBIC and friends (the `FIOBIT` format).
 def imm_port5 : Operand<i8> {
   let EncoderMethod = "encodeImm<AVR::fixup_port5, 0>";
 }
 
 /// A 6-bit port number used in the `IN` instruction and friends (the
 /// `FIORdA` format.
 def imm_port6 : Operand<i8> {
   let EncoderMethod = "encodeImm<AVR::fixup_port6, 0>";
 }
 
 // Addressing mode pattern reg+imm6
 def addr : ComplexPattern<iPTR, 2, "SelectAddr", [], [SDNPWantRoot]>;
 
 // AsmOperand class for a pointer register.
 // Used with the LD/ST family of instructions.
 // See FSTLD in AVRInstrFormats.td
 def PtrRegAsmOperand : AsmOperandClass { let Name = "Reg"; }
 
 // A special operand type for the LD/ST instructions.
 // It converts the pointer register number into a two-bit field used in the
 // instruction.
 def LDSTPtrReg : Operand<i16> {
   let MIOperandInfo = (ops PTRREGS);
   let EncoderMethod = "encodeLDSTPtrReg";
 
   let ParserMatchClass = PtrRegAsmOperand;
 }
 
 // A special operand type for the LDD/STD instructions.
 // It behaves identically to the LD/ST version, except restricts
 // the pointer registers to Y and Z.
 def LDDSTDPtrReg : Operand<i16> {
   let MIOperandInfo = (ops PTRDISPREGS);
   let EncoderMethod = "encodeLDSTPtrReg";
 
   let ParserMatchClass = PtrRegAsmOperand;
 }
 
 //===----------------------------------------------------------------------===//
 // AVR predicates for subtarget features
 //===----------------------------------------------------------------------===//
 
 def HasSRAM : Predicate<"Subtarget->hasSRAM()">,
               AssemblerPredicate<(all_of FeatureSRAM)>;
 
 def HasJMPCALL : Predicate<"Subtarget->hasJMPCALL()">,
                  AssemblerPredicate<(all_of FeatureJMPCALL)>;
 
 def HasIJMPCALL : Predicate<"Subtarget->hasIJMPCALL()">,
                   AssemblerPredicate<(all_of FeatureIJMPCALL)>;
 
 def HasEIJMPCALL : Predicate<"Subtarget->hasEIJMPCALL()">,
                    AssemblerPredicate<(all_of FeatureEIJMPCALL)>;
 
 def HasADDSUBIW : Predicate<"Subtarget->hasADDSUBIW()">,
                   AssemblerPredicate<(all_of FeatureADDSUBIW)>;
 
 def HasSmallStack : Predicate<"Subtarget->HasSmallStack()">,
                     AssemblerPredicate<(all_of FeatureSmallStack)>;
 
 def HasMOVW : Predicate<"Subtarget->hasMOVW()">,
               AssemblerPredicate<(all_of FeatureMOVW)>;
 
 def HasLPM : Predicate<"Subtarget->hasLPM()">,
              AssemblerPredicate<(all_of FeatureLPM)>;
 
 def HasLPMX : Predicate<"Subtarget->hasLPMX()">,
               AssemblerPredicate<(all_of FeatureLPMX)>;
 
 def HasELPM : Predicate<"Subtarget->hasELPM()">,
               AssemblerPredicate<(all_of FeatureELPM)>;
 
 def HasELPMX : Predicate<"Subtarget->hasELPMX()">,
                AssemblerPredicate<(all_of FeatureELPMX)>;
 
 def HasSPM : Predicate<"Subtarget->hasSPM()">,
              AssemblerPredicate<(all_of FeatureSPM)>;
 
 def HasSPMX : Predicate<"Subtarget->hasSPMX()">,
               AssemblerPredicate<(all_of FeatureSPMX)>;
 
 def HasDES : Predicate<"Subtarget->hasDES()">,
              AssemblerPredicate<(all_of FeatureDES)>;
 
 def SupportsRMW : Predicate<"Subtarget->supportsRMW()">,
                   AssemblerPredicate<(all_of FeatureRMW)>;
 
 def SupportsMultiplication : Predicate<"Subtarget->supportsMultiplication()">,
                              AssemblerPredicate<(all_of FeatureMultiplication)>;
 
 def HasBREAK : Predicate<"Subtarget->hasBREAK()">,
                AssemblerPredicate<(all_of FeatureBREAK)>;
 
 def HasTinyEncoding : Predicate<"Subtarget->hasTinyEncoding()">,
                       AssemblerPredicate<(all_of FeatureTinyEncoding)>;
 
 // AVR specific condition code. These correspond to AVR_*_COND in
 // AVRInstrInfo.td. They must be kept in synch.
 def AVR_COND_EQ : PatLeaf<(i8 0)>;
 def AVR_COND_NE : PatLeaf<(i8 1)>;
 def AVR_COND_GE : PatLeaf<(i8 2)>;
 def AVR_COND_LT : PatLeaf<(i8 3)>;
 def AVR_COND_SH : PatLeaf<(i8 4)>;
 def AVR_COND_LO : PatLeaf<(i8 5)>;
 def AVR_COND_MI : PatLeaf<(i8 6)>;
 def AVR_COND_PL : PatLeaf<(i8 7)>;
 
 //===----------------------------------------------------------------------===//
 //===----------------------------------------------------------------------===//
 // AVR Instruction list
 //===----------------------------------------------------------------------===//
 //===----------------------------------------------------------------------===//
 
 // ADJCALLSTACKDOWN/UP implicitly use/def SP because they may be expanded into
 // a stack adjustment and the codegen must know that they may modify the stack
 // pointer before prolog-epilog rewriting occurs.
 // Pessimistically assume ADJCALLSTACKDOWN / ADJCALLSTACKUP will become
 // sub / add which can clobber SREG.
 let Defs = [SP, SREG], Uses = [SP] in {
   def ADJCALLSTACKDOWN : Pseudo<(outs),
                                 (ins i16imm
                                  : $amt, i16imm
                                  : $amt2),
                                 "#ADJCALLSTACKDOWN", [(AVRcallseq_start timm
                                                        : $amt, timm
                                                        : $amt2)]>;
 
   // R31R30 is used to update SP. It is normally free because it is a
   // call-clobbered register but it is necessary to set it as a def as the
   // register allocator might use it in rare cases (for rematerialization, it
   // seems). hasSideEffects needs to be set to true so this instruction isn't
   // considered dead.
   let Defs = [R31R30], hasSideEffects = 1 in def ADJCALLSTACKUP
       : Pseudo<(outs),
                (ins i16imm
                 : $amt1, i16imm
                 : $amt2),
                "#ADJCALLSTACKUP", [(AVRcallseq_end timm
                                     : $amt1, timm
                                     : $amt2)]>;
 }
 
 //===----------------------------------------------------------------------===//
 // Addition
 //===----------------------------------------------------------------------===//
 let isCommutable = 1, Constraints = "$src = $rd", Defs = [SREG] in {
   // ADD Rd, Rr
   // Adds two 8-bit registers.
   def ADDRdRr
       : FRdRr<0b0000, 0b11,
               (outs GPR8
                : $rd),
               (ins GPR8
                : $src, GPR8
                : $rr),
               "add\t$rd, $rr",
               [(set i8
                 : $rd, (add i8
                         : $src, i8
                         : $rr)),
                (implicit SREG)]>;
 
   // ADDW Rd+1:Rd, Rr+1:Rr
   // Pseudo instruction to add four 8-bit registers as two 16-bit values.
   //
   // Expands to:
   // add Rd,    Rr
   // adc Rd+1, Rr+1
   def ADDWRdRr
       : Pseudo<(outs DREGS
                 : $rd),
                (ins DREGS
                 : $src, DREGS
                 : $rr),
                "addw\t$rd, $rr",
                [(set i16
                  : $rd, (add i16
                          : $src, i16
                          : $rr)),
                 (implicit SREG)]>;
 
   // ADC Rd, Rr
   // Adds two 8-bit registers with carry.
   let Uses = [SREG] in def ADCRdRr
       : FRdRr<0b0001, 0b11,
               (outs GPR8
                : $rd),
               (ins GPR8
                : $src, GPR8
                : $rr),
               "adc\t$rd, $rr",
               [(set i8
                 : $rd, (adde i8
                         : $src, i8
                         : $rr)),
                (implicit SREG)]>;
 
   // ADCW Rd+1:Rd, Rr+1:Rr
   // Pseudo instruction to add four 8-bit registers as two 16-bit values with
   // carry.
   //
   // Expands to:
   // adc Rd,   Rr
   // adc Rd+1, Rr+1
   let Uses = [SREG] in def ADCWRdRr : Pseudo<(outs DREGS
                                               : $rd),
                                              (ins DREGS
                                               : $src, DREGS
                                               : $rr),
                                              "adcw\t$rd, $rr", [
                                                (set i16
                                                 : $rd, (adde i16
                                                         : $src, i16
                                                         : $rr)),
                                                (implicit SREG)
                                              ]>;
 
   // AIDW Rd, k
   // Adds an immediate 6-bit value K to Rd, placing the result in Rd.
   def ADIWRdK
       : FWRdK<0b0,
               (outs IWREGS
                : $rd),
               (ins IWREGS
                : $src, imm_arith6
                : $k),
               "adiw\t$rd, $k",
               [(set i16
                 : $rd, (add i16
                         : $src, uimm6
                         : $k)),
                (implicit SREG)]>,
         Requires<[HasADDSUBIW]>;
 }
 
 //===----------------------------------------------------------------------===//
 // Subtraction
 //===----------------------------------------------------------------------===//
 let Constraints = "$src = $rd", Defs = [SREG] in {
   // SUB Rd, Rr
   // Subtracts the 8-bit value of Rr from Rd and places the value in Rd.
   def SUBRdRr
       : FRdRr<0b0001, 0b10,
               (outs GPR8
                : $rd),
               (ins GPR8
                : $src, GPR8
                : $rr),
               "sub\t$rd, $rr",
               [(set i8
                 : $rd, (sub i8
                         : $src, i8
                         : $rr)),
                (implicit SREG)]>;
 
   // SUBW Rd+1:Rd, Rr+1:Rr
   // Subtracts two 16-bit values and places the result into Rd.
   //
   // Expands to:
   // sub Rd,   Rr
   // sbc Rd+1, Rr+1
   def SUBWRdRr
       : Pseudo<(outs DREGS
                 : $rd),
                (ins DREGS
                 : $src, DREGS
                 : $rr),
                "subw\t$rd, $rr",
                [(set i16
                  : $rd, (sub i16
                          : $src, i16
                          : $rr)),
                 (implicit SREG)]>;
 
   def SUBIRdK
       : FRdK<0b0101,
              (outs LD8
               : $rd),
              (ins LD8
               : $src, imm_ldi8
               : $k),
              "subi\t$rd, $k",
              [(set i8
                : $rd, (sub i8
                        : $src, imm
                        : $k)),
               (implicit SREG)]>;
 
   // SUBIW Rd+1:Rd, K+1:K
   //
   // Expands to:
   // subi Rd,   K
   // sbci Rd+1, K+1
   def SUBIWRdK
       : Pseudo<(outs DLDREGS
                 : $rd),
                (ins DLDREGS
                 : $src, i16imm
                 : $rr),
                "subiw\t$rd, $rr",
                [(set i16
                  : $rd, (sub i16
                          : $src, imm
                          : $rr)),
                 (implicit SREG)]>;
 
   def SBIWRdK
       : FWRdK<0b1,
               (outs IWREGS
                : $rd),
               (ins IWREGS
                : $src, imm_arith6
                : $k),
               "sbiw\t$rd, $k",
               [(set i16
                 : $rd, (sub i16
                         : $src, uimm6
                         : $k)),
                (implicit SREG)]>,
         Requires<[HasADDSUBIW]>;
 
   // Subtract with carry operations which must read the carry flag in SREG.
   let Uses = [SREG] in {
     def SBCRdRr
         : FRdRr<0b0000, 0b10,
                 (outs GPR8
                  : $rd),
                 (ins GPR8
                  : $src, GPR8
                  : $rr),
                 "sbc\t$rd, $rr",
                 [(set i8
                   : $rd, (sube i8
                           : $src, i8
                           : $rr)),
                  (implicit SREG)]>;
 
     // SBCW Rd+1:Rd, Rr+1:Rr
     //
     // Expands to:
     // sbc Rd,   Rr
     // sbc Rd+1, Rr+1
     def SBCWRdRr : Pseudo<(outs DREGS
                            : $rd),
                           (ins DREGS
                            : $src, DREGS
                            : $rr),
                           "sbcw\t$rd, $rr", [
                             (set i16
                              : $rd, (sube i16
                                      : $src, i16
                                      : $rr)),
                             (implicit SREG)
                           ]>;
 
     def SBCIRdK
         : FRdK<0b0100,
                (outs LD8
                 : $rd),
                (ins LD8
                 : $src, imm_ldi8
                 : $k),
                "sbci\t$rd, $k",
                [(set i8
                  : $rd, (sube i8
                          : $src, imm
                          : $k)),
                 (implicit SREG)]>;
 
     // SBCIW Rd+1:Rd, K+1:K
     // sbci Rd,   K
     // sbci Rd+1, K+1
     def SBCIWRdK : Pseudo<(outs DLDREGS
                            : $rd),
                           (ins DLDREGS
                            : $src, i16imm
                            : $rr),
                           "sbciw\t$rd, $rr", [
                             (set i16
                              : $rd, (sube i16
                                      : $src, imm
                                      : $rr)),
                             (implicit SREG)
                           ]>;
   }
 }
 
 //===----------------------------------------------------------------------===//
 // Increment and Decrement
 //===----------------------------------------------------------------------===//
 let Constraints = "$src = $rd", Defs = [SREG] in {
   def INCRd
       : FRd<0b1001, 0b0100011,
             (outs GPR8
              : $rd),
             (ins GPR8
              : $src),
             "inc\t$rd", [(set i8
                           : $rd, (add i8
                                   : $src, 1)),
                          (implicit SREG)]>;
 
   def DECRd
       : FRd<0b1001, 0b0101010,
             (outs GPR8
              : $rd),
             (ins GPR8
              : $src),
             "dec\t$rd", [(set i8
                           : $rd, (add i8
                                   : $src, -1)),
                          (implicit SREG)]>;
 }
 
 //===----------------------------------------------------------------------===//
 // Multiplication
 //===----------------------------------------------------------------------===//
 
 let isCommutable = 1, Defs = [R1, R0, SREG] in {
   // MUL Rd, Rr
   // Multiplies Rd by Rr and places the result into R1:R0.
   let usesCustomInserter = 1 in {
     def MULRdRr : FRdRr<0b1001, 0b11, (outs),
                         (ins GPR8
                          : $lhs, GPR8
                          : $rhs),
                         "mul\t$lhs, $rhs",
                         [/*(set R1, R0, (smullohi i8:$lhs, i8:$rhs))*/]>,
                   Requires<[SupportsMultiplication]>;
 
     def MULSRdRr : FMUL2RdRr<0, (outs),
                              (ins LD8
                               : $lhs, LD8
                               : $rhs),
                              "muls\t$lhs, $rhs", []>,
                    Requires<[SupportsMultiplication]>;
   }
 
   def MULSURdRr : FMUL2RdRr<1, (outs),
                             (ins LD8lo
                              : $lhs, LD8lo
                              : $rhs),
                             "mulsu\t$lhs, $rhs", []>,
                   Requires<[SupportsMultiplication]>;
 
   def FMUL : FFMULRdRr<0b01, (outs),
                        (ins LD8lo
                         : $lhs, LD8lo
                         : $rhs),
                        "fmul\t$lhs, $rhs", []>,
              Requires<[SupportsMultiplication]>;
 
   def FMULS : FFMULRdRr<0b10, (outs),
                         (ins LD8lo
                          : $lhs, LD8lo
                          : $rhs),
                         "fmuls\t$lhs, $rhs", []>,
               Requires<[SupportsMultiplication]>;
 
   def FMULSU : FFMULRdRr<0b11, (outs),
                          (ins LD8lo
                           : $lhs, LD8lo
                           : $rhs),
                          "fmulsu\t$lhs, $rhs", []>,
                Requires<[SupportsMultiplication]>;
 }
 
 let Defs =
     [R15, R14, R13, R12, R11, R10, R9, R8, R7, R6, R5, R4, R3, R2, R1,
      R0] in def DESK : FDES<(outs),
                             (ins i8imm
                              : $k),
                             "des\t$k", []>,
     Requires<[HasDES]>;
 
 //===----------------------------------------------------------------------===//
 // Logic
 //===----------------------------------------------------------------------===//
 let Constraints = "$src = $rd", Defs = [SREG] in {
   // Register-Register logic instructions (which have the
   // property of commutativity).
   let isCommutable = 1 in {
     def ANDRdRr
         : FRdRr<0b0010, 0b00,
                 (outs GPR8
                  : $rd),
                 (ins GPR8
                  : $src, GPR8
                  : $rr),
                 "and\t$rd, $rr",
                 [(set i8
                   : $rd, (and i8
                           : $src, i8
                           : $rr)),
                  (implicit SREG)]>;
 
     // ANDW Rd+1:Rd, Rr+1:Rr
     //
     // Expands to:
     // and Rd,   Rr
     // and Rd+1, Rr+1
     def ANDWRdRr : Pseudo<(outs DREGS
                            : $rd),
                           (ins DREGS
                            : $src, DREGS
                            : $rr),
                           "andw\t$rd, $rr", [
                             (set i16
                              : $rd, (and i16
                                      : $src, i16
                                      : $rr)),
                             (implicit SREG)
                           ]>;
 
     def ORRdRr
         : FRdRr<0b0010, 0b10,
                 (outs GPR8
                  : $rd),
                 (ins GPR8
                  : $src, GPR8
                  : $rr),
                 "or\t$rd, $rr",
                 [(set i8
                   : $rd, (or i8
                           : $src, i8
                           : $rr)),
                  (implicit SREG)]>;
 
     // ORW Rd+1:Rd, Rr+1:Rr
     //
     // Expands to:
     // or Rd,   Rr
     // or Rd+1, Rr+1
     def ORWRdRr : Pseudo<(outs DREGS
                           : $rd),
                          (ins DREGS
                           : $src, DREGS
                           : $rr),
                          "orw\t$rd, $rr", [
                            (set i16
                             : $rd, (or i16
                                     : $src, i16
                                     : $rr)),
                            (implicit SREG)
                          ]>;
 
     def EORRdRr
         : FRdRr<0b0010, 0b01,
                 (outs GPR8
                  : $rd),
                 (ins GPR8
                  : $src, GPR8
                  : $rr),
                 "eor\t$rd, $rr",
                 [(set i8
                   : $rd, (xor i8
                           : $src, i8
                           : $rr)),
                  (implicit SREG)]>;
 
     // EORW Rd+1:Rd, Rr+1:Rr
     //
     // Expands to:
     // eor Rd,   Rr
     // eor Rd+1, Rr+1
     def EORWRdRr : Pseudo<(outs DREGS
                            : $rd),
                           (ins DREGS
                            : $src, DREGS
                            : $rr),
                           "eorw\t$rd, $rr", [
                             (set i16
                              : $rd, (xor i16
                                      : $src, i16
                                      : $rr)),
                             (implicit SREG)
                           ]>;
   }
 
   def ANDIRdK
       : FRdK<0b0111,
              (outs LD8
               : $rd),
              (ins LD8
               : $src, imm_ldi8
               : $k),
              "andi\t$rd, $k",
              [(set i8
                : $rd, (and i8
                        : $src, imm
                        : $k)),
               (implicit SREG)]>;
 
   // ANDI Rd+1:Rd, K+1:K
   //
   // Expands to:
   // andi Rd,   K
   // andi Rd+1, K+1
   def ANDIWRdK
       : Pseudo<(outs DLDREGS
                 : $rd),
                (ins DLDREGS
                 : $src, i16imm
                 : $k),
                "andiw\t$rd, $k",
                [(set i16
                  : $rd, (and i16
                          : $src, imm
                          : $k)),
                 (implicit SREG)]>;
 
   def ORIRdK
       : FRdK<0b0110,
              (outs LD8
               : $rd),
              (ins LD8
               : $src, imm_ldi8
               : $k),
              "ori\t$rd, $k",
              [(set i8
                : $rd, (or i8
                        : $src, imm
                        : $k)),
               (implicit SREG)]>;
 
   // ORIW Rd+1:Rd, K+1,K
   //
   // Expands to:
   // ori Rd,   K
   // ori Rd+1, K+1
   def ORIWRdK
       : Pseudo<(outs DLDREGS
                 : $rd),
                (ins DLDREGS
                 : $src, i16imm
                 : $rr),
                "oriw\t$rd, $rr",
                [(set i16
                  : $rd, (or i16
                          : $src, imm
                          : $rr)),
                 (implicit SREG)]>;
 }
 
 //===----------------------------------------------------------------------===//
 // One's/Two's Complement
 //===----------------------------------------------------------------------===//
 let Constraints = "$src = $rd", Defs = [SREG] in {
   def COMRd
       : FRd<0b1001, 0b0100000,
             (outs GPR8
              : $rd),
             (ins GPR8
              : $src),
             "com\t$rd", [(set i8
                           : $rd, (not i8
                                   : $src)),
                          (implicit SREG)]>;
 
   // COMW Rd+1:Rd
   //
   // Expands to:
   // com Rd
   // com Rd+1
   def COMWRd : Pseudo<(outs DREGS
                        : $rd),
                       (ins DREGS
                        : $src),
                       "comw\t$rd",
                       [(set i16
                         : $rd, (not i16
                                 : $src)),
                        (implicit SREG)]>;
 
   def NEGRd
       : FRd<0b1001, 0b0100001,
             (outs GPR8
              : $rd),
             (ins GPR8
              : $src),
             "neg\t$rd", [(set i8
                           : $rd, (ineg i8
                                   : $src)),
                          (implicit SREG)]>;
 
   // NEGW Rd+1:Rd
   //
   // Expands to:
   // neg Rd+1
   // neg Rd
   // sbc Rd+1, r1
   def NEGWRd : Pseudo<(outs DREGS
                        : $rd),
                       (ins DREGS
                        : $src),
                       "negw\t$rd",
                       [(set i16
                         : $rd, (ineg i16
                                 : $src)),
                        (implicit SREG)]>;
 }
 
 // TST Rd
 // Test for zero of minus.
 // This operation is identical to a `Rd AND Rd`.
 def : InstAlias<"tst\t$rd", (ANDRdRr GPR8 : $rd, GPR8 : $rd)>;
 
 // SBR Rd, K
 //
 // Mnemonic alias to 'ORI Rd, K'. Same bit pattern, same operands,
 // same everything.
 def : InstAlias<"sbr\t$rd, $k",
                 (ORIRdK LD8
                  : $rd, imm_ldi8
                  : $k),
                 /* Disable display, so we don't override ORI */ 0>;
 
 //===----------------------------------------------------------------------===//
 // Jump instructions
 //===----------------------------------------------------------------------===//
 let isBarrier = 1, isBranch = 1, isTerminator = 1 in {
   def RJMPk : FBRk<0, (outs),
                    (ins brtarget_13
                     : $target),
                    "rjmp\t$target", [(br bb
                                       : $target)]>;
 
   let isIndirectBranch = 1,
       Uses = [R31R30] in def IJMP
       : F16<0b1001010000001001, (outs), (ins), "ijmp", []>,
       Requires<[HasIJMPCALL]>;
 
   let isIndirectBranch = 1,
       Uses = [R31R30] in def EIJMP
       : F16<0b1001010000011001, (outs), (ins), "eijmp", []>,
       Requires<[HasEIJMPCALL]>;
 
   def JMPk : F32BRk<0b110, (outs),
                     (ins call_target
                      : $k),
                     "jmp\t$k", []>,
              Requires<[HasJMPCALL]>;
 }
 
 //===----------------------------------------------------------------------===//
 // Call instructions
 //===----------------------------------------------------------------------===//
 let isCall = 1 in {
   // SP is marked as a use to prevent stack-pointer assignments that appear
   // immediately before calls from potentially appearing dead.
-  let Uses = [SP] in def RCALLk : FBRk<1, (outs),
-                                       (ins brtarget_13
-                                        : $target),
-                                       "rcall\t$target", []>;
+  let Uses = [SP] in def RCALLk : FBRk<1, (outs), (ins rcalltarget_13:$k),
+                                       "rcall\t$k", [(AVRcall imm:$k)]>;
 
   // SP is marked as a use to prevent stack-pointer assignments that appear
   // immediately before calls from potentially appearing dead.
   let Uses = [SP, R31R30] in def ICALL
       : F16<0b1001010100001001, (outs), (ins variable_ops), "icall", []>,
       Requires<[HasIJMPCALL]>;
 
   // SP is marked as a use to prevent stack-pointer assignments that appear
   // immediately before calls from potentially appearing dead.
   let Uses = [SP, R31R30] in def EICALL
       : F16<0b1001010100011001, (outs), (ins variable_ops), "eicall", []>,
       Requires<[HasEIJMPCALL]>;
 
   // SP is marked as a use to prevent stack-pointer assignments that appear
   // immediately before calls from potentially appearing dead.
   //
-  //: TODO: the imm field can be either 16 or 22 bits in devices with more
+  // TODO: the imm field can be either 16 or 22 bits in devices with more
   // than 64k of ROM, fix it once we support the largest devices.
-  let Uses = [SP] in def CALLk : F32BRk<0b111, (outs),
-                                        (ins call_target
-                                         : $k),
-                                        "call\t$k", [(AVRcall imm
-                                                      : $k)]>,
+  let Uses = [SP] in def CALLk : F32BRk<0b111, (outs), (ins call_target:$k),
+                                        "call\t$k", [(AVRcall imm:$k)]>,
       Requires<[HasJMPCALL]>;
 }
 
 //===----------------------------------------------------------------------===//
 // Return instructions.
 //===----------------------------------------------------------------------===//
 let isTerminator = 1, isReturn = 1, isBarrier = 1 in {
   def RET : F16<0b1001010100001000, (outs), (ins), "ret", [(AVRretflag)]>;
 
   def RETI : F16<0b1001010100011000, (outs), (ins), "reti", [(AVRretiflag)]>;
 }
 
 //===----------------------------------------------------------------------===//
 // Compare operations.
 //===----------------------------------------------------------------------===//
 let Defs = [SREG] in {
   // CPSE Rd, Rr
   // Compare Rd and Rr, skipping the next instruction if they are equal.
   let isBarrier = 1, isBranch = 1,
       isTerminator = 1 in def CPSE : FRdRr<0b0001, 0b00, (outs),
                                            (ins GPR8
                                             : $rd, GPR8
                                             : $rr),
                                            "cpse\t$rd, $rr", []>;
 
   def CPRdRr
       : FRdRr<0b0001, 0b01, (outs),
               (ins GPR8
                : $rd, GPR8
                : $rr),
               "cp\t$rd, $rr", [(AVRcmp i8
                                 : $rd, i8
                                 : $rr),
                                (implicit SREG)]>;
 
   // CPW Rd+1:Rd, Rr+1:Rr
   //
   // Expands to:
   // cp  Rd,   Rr
   // cpc Rd+1, Rr+1
   def CPWRdRr : Pseudo<(outs),
                        (ins DREGS
                         : $src, DREGS
                         : $src2),
                        "cpw\t$src, $src2",
                        [(AVRcmp i16
                          : $src, i16
                          : $src2),
                         (implicit SREG)]>;
 
   let Uses = [SREG] in def CPCRdRr
       : FRdRr<0b0000, 0b01, (outs),
               (ins GPR8
                : $rd, GPR8
                : $rr),
               "cpc\t$rd, $rr", [(AVRcmpc i8
                                  : $rd, i8
                                  : $rr),
                                 (implicit SREG)]>;
 
   // CPCW Rd+1:Rd. Rr+1:Rr
   //
   // Expands to:
   // cpc Rd,   Rr
   // cpc Rd+1, Rr+1
   let Uses = [SREG] in def CPCWRdRr
       : Pseudo<(outs),
                (ins DREGS
                 : $src, DREGS
                 : $src2),
                "cpcw\t$src, $src2",
                [(AVRcmpc i16
                  : $src, i16
                  : $src2),
                 (implicit SREG)]>;
 
   // CPI Rd, K
   // Compares a register with an 8 bit immediate.
   def CPIRdK
       : FRdK<0b0011, (outs),
              (ins LD8
               : $rd, imm_ldi8
               : $k),
              "cpi\t$rd, $k", [(AVRcmp i8
                                : $rd, imm
                                : $k),
                               (implicit SREG)]>;
 }
 
 //===----------------------------------------------------------------------===//
 // Register conditional skipping/branching operations.
 //===----------------------------------------------------------------------===//
 let isBranch = 1, isTerminator = 1 in {
   // Conditional skipping on GPR register bits, and
   // conditional skipping on IO register bits.
   let isBarrier = 1 in {
     def SBRCRrB : FRdB<0b10, (outs),
                        (ins GPR8
                         : $rr, i8imm
                         : $b),
                        "sbrc\t$rr, $b", []>;
 
     def SBRSRrB : FRdB<0b11, (outs),
                        (ins GPR8
                         : $rr, i8imm
                         : $b),
                        "sbrs\t$rr, $b", []>;
 
     def SBICAb : FIOBIT<0b01, (outs),
                         (ins imm_port5
                          : $a, i8imm
                          : $b),
                         "sbic\t$a, $b", []>;
 
     def SBISAb : FIOBIT<0b11, (outs),
                         (ins imm_port5
                          : $a, i8imm
                          : $b),
                         "sbis\t$a, $b", []>;
   }
 
   // Relative branches on status flag bits.
   let Uses = [SREG] in {
     // BRBS s, k
     // Branch if `s` flag in status register is set.
     def BRBSsk : FSK<0, (outs),
                      (ins i8imm
                       : $s, relbrtarget_7
                       : $k),
                      "brbs\t$s, $k", []>;
 
     // BRBC s, k
     // Branch if `s` flag in status register is clear.
     def BRBCsk : FSK<1, (outs),
                      (ins i8imm
                       : $s, relbrtarget_7
                       : $k),
                      "brbc\t$s, $k", []>;
   }
 }
 
 // BRCS k
 // Branch if carry flag is set
 def : InstAlias<"brcs\t$k", (BRBSsk 0, relbrtarget_7 : $k)>;
 
 // BRCC k
 // Branch if carry flag is clear
 def : InstAlias<"brcc\t$k", (BRBCsk 0, relbrtarget_7 : $k)>;
 
 // BRHS k
 // Branch if half carry flag is set
 def : InstAlias<"brhs\t$k", (BRBSsk 5, relbrtarget_7 : $k)>;
 
 // BRHC k
 // Branch if half carry flag is clear
 def : InstAlias<"brhc\t$k", (BRBCsk 5, relbrtarget_7 : $k)>;
 
 // BRTS k
 // Branch if the T flag is set
 def : InstAlias<"brts\t$k", (BRBSsk 6, relbrtarget_7 : $k)>;
 
 // BRTC k
 // Branch if the T flag is clear
 def : InstAlias<"brtc\t$k", (BRBCsk 6, relbrtarget_7 : $k)>;
 
 // BRVS k
 // Branch if the overflow flag is set
 def : InstAlias<"brvs\t$k", (BRBSsk 3, relbrtarget_7 : $k)>;
 
 // BRVC k
 // Branch if the overflow flag is clear
 def : InstAlias<"brvc\t$k", (BRBCsk 3, relbrtarget_7 : $k)>;
 
 // BRIE k
 // Branch if the global interrupt flag is enabled
 def : InstAlias<"brie\t$k", (BRBSsk 7, relbrtarget_7 : $k)>;
 
 // BRID k
 // Branch if the global interrupt flag is disabled
 def : InstAlias<"brid\t$k", (BRBCsk 7, relbrtarget_7 : $k)>;
 
 //===----------------------------------------------------------------------===//
 // PC-relative conditional branches
 //===----------------------------------------------------------------------===//
 // Based on status register. We cannot simplify these into instruction aliases
 // because we also need to be able to specify a pattern to match for ISel.
 let isBranch = 1, isTerminator = 1, Uses = [SREG] in {
   def BREQk : FBRsk<0, 0b001, (outs),
                     (ins relbrtarget_7
                      : $target),
                     "breq\t$target", [(AVRbrcond bb
                                        : $target, AVR_COND_EQ)]>;
 
   def BRNEk : FBRsk<1, 0b001, (outs),
                     (ins relbrtarget_7
                      : $target),
                     "brne\t$target", [(AVRbrcond bb
                                        : $target, AVR_COND_NE)]>;
 
   def BRSHk : FBRsk<1, 0b000, (outs),
                     (ins relbrtarget_7
                      : $target),
                     "brsh\t$target", [(AVRbrcond bb
                                        : $target, AVR_COND_SH)]>;
 
   def BRLOk : FBRsk<0, 0b000, (outs),
                     (ins relbrtarget_7
                      : $target),
                     "brlo\t$target", [(AVRbrcond bb
                                        : $target, AVR_COND_LO)]>;
 
   def BRMIk : FBRsk<0, 0b010, (outs),
                     (ins relbrtarget_7
                      : $target),
                     "brmi\t$target", [(AVRbrcond bb
                                        : $target, AVR_COND_MI)]>;
 
   def BRPLk : FBRsk<1, 0b010, (outs),
                     (ins relbrtarget_7
                      : $target),
                     "brpl\t$target", [(AVRbrcond bb
                                        : $target, AVR_COND_PL)]>;
 
   def BRGEk : FBRsk<1, 0b100, (outs),
                     (ins relbrtarget_7
                      : $target),
                     "brge\t$target", [(AVRbrcond bb
                                        : $target, AVR_COND_GE)]>;
 
   def BRLTk : FBRsk<0, 0b100, (outs),
                     (ins relbrtarget_7
                      : $target),
                     "brlt\t$target", [(AVRbrcond bb
                                        : $target, AVR_COND_LT)]>;
 }
 
 //===----------------------------------------------------------------------===//
 // Data transfer instructions
 //===----------------------------------------------------------------------===//
 // 8 and 16-bit register move instructions.
 let hasSideEffects = 0 in {
   def MOVRdRr : FRdRr<0b0010, 0b11,
                       (outs GPR8
                        : $rd),
                       (ins GPR8
                        : $rr),
                       "mov\t$rd, $rr", []>;
 
   def MOVWRdRr : FMOVWRdRr<(outs DREGS
                             : $dst),
                            (ins DREGS
                             : $src),
                            "movw\t$dst, $src", []>,
                  Requires<[HasMOVW]>;
 }
 
 // Load immediate values into registers.
 let isReMaterializable = 1 in {
   def LDIRdK : FRdK<0b1110,
                     (outs LD8
                      : $rd),
                     (ins imm_ldi8
                      : $k),
                     "ldi\t$rd, $k", [(set i8
                                       : $rd, imm
                                       : $k)]>;
 
   // LDIW Rd+1:Rd, K+1:K
   //
   // Expands to:
   // ldi Rd,   K
   // ldi Rd+1, K+1
   def LDIWRdK : Pseudo<(outs DLDREGS
                         : $dst),
                        (ins i16imm
                         : $src),
                        "ldiw\t$dst, $src", [(set i16
                                              : $dst, imm
                                              : $src)]>;
 }
 
 // Load from data space into register.
 let canFoldAsLoad = 1, isReMaterializable = 1 in {
   def LDSRdK : F32DM<0b0,
                      (outs GPR8
                       : $rd),
                      (ins imm16
                       : $k),
                      "lds\t$rd, $k", [(set i8
                                        : $rd, (load imm
                                                : $k))]>,
                Requires<[HasSRAM]>;
 
   // LDSW Rd+1:Rd, K+1:K
   //
   // Expands to:
   // lds Rd,  (K+1:K)
   // lds Rd+1 (K+1:K) + 1
   def LDSWRdK : Pseudo<(outs DREGS
                         : $dst),
                        (ins i16imm
                         : $src),
                        "ldsw\t$dst, $src", [(set i16
                                              : $dst, (load imm
                                                       : $src))]>,
                 Requires<[HasSRAM]>;
 }
 
 // Indirect loads.
 let canFoldAsLoad = 1, isReMaterializable = 1 in {
   def LDRdPtr : FSTLD<0, 0b00,
                       (outs GPR8
                        : $reg),
                       (ins LDSTPtrReg
                        : $ptrreg),
                       "ld\t$reg, $ptrreg", [(set GPR8
                                              : $reg, (load i16
                                                       : $ptrreg))]>,
                 Requires<[HasSRAM]>;
 
   // LDW Rd+1:Rd, P
   //
   // Expands to:
   // ld  Rd,   P
   // ldd Rd+1, P+1
   let Constraints = "@earlyclobber $reg" in def LDWRdPtr
       : Pseudo<(outs DREGS
                 : $reg),
                (ins PTRDISPREGS
                 : $ptrreg),
                "ldw\t$reg, $ptrreg", [(set i16
                                        : $reg, (load i16
                                                 : $ptrreg))]>,
       Requires<[HasSRAM]>;
 }
 
 // Indirect loads (with postincrement or predecrement).
 let mayLoad = 1, hasSideEffects = 0,
     Constraints = "$ptrreg = $base_wb,@earlyclobber $reg" in {
   def LDRdPtrPi : FSTLD<0, 0b01,
                         (outs GPR8
                          : $reg, PTRREGS
                          : $base_wb),
                         (ins LDSTPtrReg
                          : $ptrreg),
                         "ld\t$reg, $ptrreg+", []>,
                   Requires<[HasSRAM]>;
 
   // LDW Rd+1:Rd, P+
   // Expands to:
   // ld Rd,   P+
   // ld Rd+1, P+
   def LDWRdPtrPi : Pseudo<(outs DREGS
                            : $reg, PTRREGS
                            : $base_wb),
                           (ins PTRREGS
                            : $ptrreg),
                           "ldw\t$reg, $ptrreg+", []>,
                    Requires<[HasSRAM]>;
 
   def LDRdPtrPd : FSTLD<0, 0b10,
                         (outs GPR8
                          : $reg, PTRREGS
                          : $base_wb),
                         (ins LDSTPtrReg
                          : $ptrreg),
                         "ld\t$reg, -$ptrreg", []>,
                   Requires<[HasSRAM]>;
 
   // LDW Rd+1:Rd, -P
   //
   // Expands to:
   // ld Rd+1, -P
   // ld Rd,   -P
   def LDWRdPtrPd : Pseudo<(outs DREGS
                            : $reg, PTRREGS
                            : $base_wb),
                           (ins PTRREGS
                            : $ptrreg),
                           "ldw\t$reg, -$ptrreg", []>,
                    Requires<[HasSRAM]>;
 }
 
 // Load indirect with displacement operations.
 let canFoldAsLoad = 1, isReMaterializable = 1 in {
   let Constraints = "@earlyclobber $reg" in def LDDRdPtrQ
       : FSTDLDD<0,
                 (outs GPR8
                  : $reg),
                 (ins memri
                  : $memri),
                 "ldd\t$reg, $memri", [(set i8
                                        : $reg, (load addr
                                                 : $memri))]>,
       Requires<[HasSRAM]>;
 
   // LDDW Rd+1:Rd, P+q
   //
   // Expands to:
   // ldd Rd,   P+q
   // ldd Rd+1, P+q+1
   let Constraints = "@earlyclobber $dst" in def LDDWRdPtrQ
       : Pseudo<(outs DREGS
                 : $dst),
                (ins memri
                 : $memri),
                "lddw\t$dst, $memri", [(set i16
                                        : $dst, (load addr
                                                 : $memri))]>,
       Requires<[HasSRAM]>;
 
   // An identical pseudo instruction to LDDWRdPtrQ, expect restricted to the Y
   // register and without the @earlyclobber flag.
   //
   // Used to work around a bug caused by the register allocator not
   // being able to handle the expansion of a COPY into an machine instruction
   // that has an earlyclobber flag. This is because the register allocator will
   // try expand a copy from a register slot into an earlyclobber instruction.
   // Instructions that are earlyclobber need to be in a dedicated earlyclobber
   // slot.
   //
   // This pseudo instruction can be used pre-AVR pseudo expansion in order to
   // get a frame index load without directly using earlyclobber instructions.
   //
   // The pseudo expansion pass trivially expands this into LDDWRdPtrQ.
   //
   // This instruction may be removed once PR13375 is fixed.
   let mayLoad = 1,
       hasSideEffects = 0 in def LDDWRdYQ : Pseudo<(outs DREGS
                                                    : $dst),
                                                   (ins memri
                                                    : $memri),
                                                   "lddw\t$dst, $memri", []>,
       Requires<[HasSRAM]>;
 }
 
 class AtomicLoad<PatFrag Op, RegisterClass DRC, RegisterClass PTRRC>
     : Pseudo<(outs DRC
               : $rd),
              (ins PTRRC
               : $rr),
              "atomic_op", [(set DRC
                             : $rd, (Op i16
                                     : $rr))]>;
 
 class AtomicStore<PatFrag Op, RegisterClass DRC, RegisterClass PTRRC>
     : Pseudo<(outs),
              (ins PTRRC
               : $rd, DRC
               : $rr),
              "atomic_op", [(Op i16
                             : $rd, DRC
                             : $rr)]>;
 
 let Constraints =
     "@earlyclobber $rd" in class AtomicLoadOp<PatFrag Op, RegisterClass DRC,
                                               RegisterClass PTRRC>
     : Pseudo<(outs DRC
               : $rd),
              (ins PTRRC
               : $rr, DRC
               : $operand),
              "atomic_op", [(set DRC
                             : $rd, (Op i16
                                     : $rr, DRC
                                     : $operand))]>;
 
 // FIXME: I think 16-bit atomic binary ops need to mark
 // r0 as clobbered.
 
 // Atomic instructions
 // ===================
 //
 // These are all expanded by AVRExpandPseudoInsts
 //
 // 8-bit operations can use any pointer register because
 // they are expanded directly into an LD/ST instruction.
 //
 // 16-bit operations use 16-bit load/store postincrement instructions,
 // which require PTRDISPREGS.
 
 def AtomicLoad8 : AtomicLoad<atomic_load_8, GPR8, PTRREGS>;
 def AtomicLoad16 : AtomicLoad<atomic_load_16, DREGS, PTRDISPREGS>;
 
 def AtomicStore8 : AtomicStore<atomic_store_8, GPR8, PTRREGS>;
 def AtomicStore16 : AtomicStore<atomic_store_16, DREGS, PTRDISPREGS>;
 
 class AtomicLoadOp8<PatFrag Op> : AtomicLoadOp<Op, GPR8, PTRREGS>;
 class AtomicLoadOp16<PatFrag Op> : AtomicLoadOp<Op, DREGS, PTRDISPREGS>;
 
 def AtomicLoadAdd8 : AtomicLoadOp8<atomic_load_add_8>;
 def AtomicLoadAdd16 : AtomicLoadOp16<atomic_load_add_16>;
 def AtomicLoadSub8 : AtomicLoadOp8<atomic_load_sub_8>;
 def AtomicLoadSub16 : AtomicLoadOp16<atomic_load_sub_16>;
 def AtomicLoadAnd8 : AtomicLoadOp8<atomic_load_and_8>;
 def AtomicLoadAnd16 : AtomicLoadOp16<atomic_load_and_16>;
 def AtomicLoadOr8 : AtomicLoadOp8<atomic_load_or_8>;
 def AtomicLoadOr16 : AtomicLoadOp16<atomic_load_or_16>;
 def AtomicLoadXor8 : AtomicLoadOp8<atomic_load_xor_8>;
 def AtomicLoadXor16 : AtomicLoadOp16<atomic_load_xor_16>;
 def AtomicFence
     : Pseudo<(outs), (ins), "atomic_fence", [(atomic_fence timm, timm)]>;
 
 // Indirect store from register to data space.
 def STSKRr : F32DM<0b1, (outs),
                    (ins imm16
                     : $k, GPR8
                     : $rd),
                    "sts\t$k, $rd", [(store i8
                                      : $rd, imm
                                      : $k)]>,
              Requires<[HasSRAM]>;
 
 // STSW K+1:K, Rr+1:Rr
 //
 // Expands to:
 // sts Rr+1, (K+1:K) + 1
 // sts Rr,   (K+1:K)
 def STSWKRr : Pseudo<(outs),
                      (ins i16imm
                       : $dst, DREGS
                       : $src),
                      "stsw\t$dst, $src", [(store i16
                                            : $src, imm
                                            : $dst)]>,
               Requires<[HasSRAM]>;
 
 // Indirect stores.
 // ST P, Rr
 // Stores the value of Rr into the location addressed by pointer P.
 def STPtrRr : FSTLD<1, 0b00, (outs),
                     (ins LDSTPtrReg
                      : $ptrreg, GPR8
                      : $reg),
                     "st\t$ptrreg, $reg", [(store GPR8
                                            : $reg, i16
                                            : $ptrreg)]>,
               Requires<[HasSRAM]>;
 
 // STW P, Rr+1:Rr
 // Stores the value of Rr into the location addressed by pointer P.
 //
 // Expands to:
 // st P, Rr
 // std P+1, Rr+1
 def STWPtrRr : Pseudo<(outs),
                       (ins PTRDISPREGS
                        : $ptrreg, DREGS
                        : $reg),
                       "stw\t$ptrreg, $reg", [(store i16
                                               : $reg, i16
                                               : $ptrreg)]>,
                Requires<[HasSRAM]>;
 
 // Indirect stores (with postincrement or predecrement).
 let Constraints = "$ptrreg = $base_wb,@earlyclobber $base_wb" in {
 
   // ST P+, Rr
   // Stores the value of Rr into the location addressed by pointer P.
   // Post increments P.
   def STPtrPiRr : FSTLD<1, 0b01,
                         (outs LDSTPtrReg
                          : $base_wb),
                         (ins LDSTPtrReg
                          : $ptrreg, GPR8
                          : $reg, i8imm
                          : $offs),
                         "st\t$ptrreg+, $reg", [(set i16
                                                 : $base_wb, (post_store GPR8
                                                              : $reg, i16
                                                              : $ptrreg, imm
                                                              : $offs))]>,
                   Requires<[HasSRAM]>;
 
   // STW P+, Rr+1:Rr
   // Stores the value of Rr into the location addressed by pointer P.
   // Post increments P.
   //
   // Expands to:
   // st P+, Rr
   // st P+, Rr+1
   def STWPtrPiRr : Pseudo<(outs PTRREGS
                            : $base_wb),
                           (ins PTRREGS
                            : $ptrreg, DREGS
                            : $trh, i8imm
                            : $offs),
                           "stw\t$ptrreg+, $trh", [(set PTRREGS
                                                    : $base_wb, (post_store DREGS
                                                                 : $trh, PTRREGS
                                                                 : $ptrreg, imm
                                                                 : $offs))]>,
                    Requires<[HasSRAM]>;
 
   // ST -P, Rr
   // Stores the value of Rr into the location addressed by pointer P.
   // Pre decrements P.
   def STPtrPdRr : FSTLD<1, 0b10,
                         (outs LDSTPtrReg
                          : $base_wb),
                         (ins LDSTPtrReg
                          : $ptrreg, GPR8
                          : $reg, i8imm
                          : $offs),
                         "st\t-$ptrreg, $reg", [(set i16
                                                 : $base_wb, (pre_store GPR8
                                                              : $reg, i16
                                                              : $ptrreg, imm
                                                              : $offs))]>,
                   Requires<[HasSRAM]>;
 
   // STW -P, Rr+1:Rr
   // Stores the value of Rr into the location addressed by pointer P.
   // Pre decrements P.
   //
   // Expands to:
   // st -P, Rr+1
   // st -P, Rr
   def STWPtrPdRr : Pseudo<(outs PTRREGS
                            : $base_wb),
                           (ins PTRREGS
                            : $ptrreg, DREGS
                            : $reg, i8imm
                            : $offs),
                           "stw\t-$ptrreg, $reg", [(set PTRREGS
                                                    : $base_wb, (pre_store i16
                                                                 : $reg, i16
                                                                 : $ptrreg, imm
                                                                 : $offs))]>,
                    Requires<[HasSRAM]>;
 }
 
 // Store indirect with displacement operations.
 // STD P+q, Rr
 // Stores the value of Rr into the location addressed by pointer P with a
 // displacement of q. Does not modify P.
 def STDPtrQRr : FSTDLDD<1, (outs),
                         (ins memri
                          : $memri, GPR8
                          : $reg),
                         "std\t$memri, $reg", [(store i8
                                                : $reg, addr
                                                : $memri)]>,
                 Requires<[HasSRAM]>;
 
 // STDW P+q, Rr+1:Rr
 // Stores the value of Rr into the location addressed by pointer P with a
 // displacement of q. Does not modify P.
 //
 // Expands to:
 // std P+q,   Rr
 // std P+q+1, Rr+1
 def STDWPtrQRr : Pseudo<(outs),
                         (ins memri
                          : $memri, DREGS
                          : $src),
                         "stdw\t$memri, $src", [(store i16
                                                 : $src, addr
                                                 : $memri)]>,
                  Requires<[HasSRAM]>;
 
 // Load program memory operations.
 let canFoldAsLoad = 1, isReMaterializable = 1, mayLoad = 1,
     hasSideEffects = 0 in {
   let Defs = [R0],
       Uses = [R31R30] in def LPM
       : F16<0b1001010111001000, (outs), (ins), "lpm", []>,
       Requires<[HasLPM]>;
 
   def LPMRdZ : FLPMX<0, 0,
                      (outs GPR8
                       : $dst),
                      (ins ZREG
                       : $z),
                      "lpm\t$dst, $z", []>,
                Requires<[HasLPMX]>;
 
   // Load program memory, while postincrementing the Z register.
   let Defs = [R31R30] in {
     def LPMRdZPi : FLPMX<0, 1,
                          (outs GPR8
                           : $dst),
                          (ins ZREG
                           : $z),
                          "lpm\t$dst, $z+", []>,
                    Requires<[HasLPMX]>;
 
     def LPMWRdZ : Pseudo<(outs DREGS
                           : $dst),
                          (ins ZREG
                           : $z),
                          "lpmw\t$dst, $z", []>,
                   Requires<[HasLPMX]>;
 
     def LPMWRdZPi : Pseudo<(outs DREGS
                             : $dst),
                            (ins ZREG
                             : $z),
                            "lpmw\t$dst, $z+", []>,
                     Requires<[HasLPMX]>;
   }
 }
 
 // Extended load program memory operations.
 let mayLoad = 1, hasSideEffects = 0 in {
   let Defs = [R0],
       Uses = [R31R30] in def ELPM
       : F16<0b1001010111011000, (outs), (ins), "elpm", []>,
       Requires<[HasELPM]>;
 
   def ELPMRdZ : FLPMX<1, 0, (outs GPR8:$dst), (ins ZREG:$z),
                       "elpm\t$dst, $z", []>,
                 Requires<[HasELPMX]>;
 
   let Defs = [R31R30] in {
     def ELPMRdZPi : FLPMX<1, 1, (outs GPR8:$dst), (ins ZREG:$z),
                           "elpm\t$dst, $z+", []>,
                     Requires<[HasELPMX]>;
   }
 
   // These pseudos are combination of the OUT and ELPM instructions.
   let Defs = [R31R30], hasSideEffects = 1 in {
     def ELPMBRdZ : Pseudo<(outs GPR8:$dst), (ins ZREG:$z, LD8:$p),
                           "elpmb\t$dst, $z, $p", []>,
                    Requires<[HasELPMX]>;
 
     def ELPMWRdZ : Pseudo<(outs DREGS:$dst), (ins ZREG:$z, LD8:$p),
                           "elpmw\t$dst, $z, $p", []>,
                    Requires<[HasELPMX]>;
 
     def ELPMBRdZPi : Pseudo<(outs GPR8:$dst), (ins ZREG:$z, LD8:$p),
                             "elpmb\t$dst, $z+, $p", []>,
                      Requires<[HasELPMX]>;
 
     def ELPMWRdZPi : Pseudo<(outs DREGS:$dst), (ins ZREG:$z, LD8:$p),
                             "elpmw\t$dst, $z+, $p", []>,
                      Requires<[HasELPMX]>;
   }
 }
 
 // Store program memory operations.
 let Uses = [R1, R0] in {
   let Uses = [R31R30, R1, R0] in def SPM
       : F16<0b1001010111101000, (outs), (ins), "spm", []>,
       Requires<[HasSPM]>;
 
   let Defs = [R31R30] in def SPMZPi : F16<0b1001010111111000, (outs),
                                           (ins ZREG
                                            : $z),
                                           "spm $z+", []>,
       Requires<[HasSPMX]>;
 }
 
 // Read data from IO location operations.
 let canFoldAsLoad = 1, isReMaterializable = 1 in {
   def INRdA : FIORdA<(outs GPR8
                       : $dst),
                      (ins imm_port6
                       : $src),
                      "in\t$dst, $src", [(set i8
                                          : $dst, (load ioaddr8
                                                   : $src))]>;
 
   def INWRdA : Pseudo<(outs DREGS
                        : $dst),
                       (ins imm_port6
                        : $src),
                       "inw\t$dst, $src", [(set i16
                                            : $dst, (load ioaddr16
                                                     : $src))]>;
 }
 
 // Write data to IO location operations.
 def OUTARr : FIOARr<(outs),
                     (ins imm_port6
                      : $dst, GPR8
                      : $src),
                     "out\t$dst, $src", [(store i8
                                          : $src, ioaddr8
                                          : $dst)]>;
 
 def OUTWARr : Pseudo<(outs),
                      (ins imm_port6
                       : $dst, DREGS
                       : $src),
                      "outw\t$dst, $src", [(store i16
                                            : $src, ioaddr16
                                            : $dst)]>;
 
 // Stack push/pop operations.
 let Defs = [SP], Uses = [SP], hasSideEffects = 0 in {
   // Stack push operations.
   let mayStore = 1 in {
     def PUSHRr : FRd<0b1001, 0b0011111, (outs),
                      (ins GPR8
                       : $reg),
                      "push\t$reg", []>,
                  Requires<[HasSRAM]>;
 
     def PUSHWRr : Pseudo<(outs),
                          (ins DREGS
                           : $reg),
                          "pushw\t$reg", []>,
                   Requires<[HasSRAM]>;
   }
 
   // Stack pop operations.
   let mayLoad = 1 in {
     def POPRd : FRd<0b1001, 0b0001111,
                     (outs GPR8
                      : $reg),
                     (ins), "pop\t$reg", []>,
                 Requires<[HasSRAM]>;
 
     def POPWRd : Pseudo<(outs DREGS
                          : $reg),
                         (ins), "popw\t$reg", []>,
                  Requires<[HasSRAM]>;
   }
 }
 
 // Read-Write-Modify (RMW) instructions.
 def XCHZRd : FZRd<0b100,
                   (outs GPR8
                    : $rd),
                   (ins ZREG
                    : $z),
                   "xch\t$z, $rd", []>,
              Requires<[SupportsRMW]>;
 
 def LASZRd : FZRd<0b101,
                   (outs GPR8
                    : $rd),
                   (ins ZREG
                    : $z),
                   "las\t$z, $rd", []>,
              Requires<[SupportsRMW]>;
 
 def LACZRd : FZRd<0b110,
                   (outs GPR8
                    : $rd),
                   (ins ZREG
                    : $z),
                   "lac\t$z, $rd", []>,
              Requires<[SupportsRMW]>;
 
 def LATZRd : FZRd<0b111,
                   (outs GPR8
                    : $rd),
                   (ins ZREG
                    : $z),
                   "lat\t$z, $rd", []>,
              Requires<[SupportsRMW]>;
 
 //===----------------------------------------------------------------------===//
 // Bit and bit-test instructions
 //===----------------------------------------------------------------------===//
 
 // Bit shift/rotate operations.
 let Constraints = "$src = $rd", Defs = [SREG] in {
   // 8-bit LSL is an alias of ADD Rd, Rd
 
   def LSLWRd : Pseudo<(outs DREGS
                        : $rd),
                       (ins DREGS
                        : $src),
                       "lslw\t$rd",
                       [(set i16
                         : $rd, (AVRlsl i16
                                 : $src)),
                        (implicit SREG)]>;
 
   def LSLWHiRd : Pseudo<(outs DREGS:$rd), (ins DREGS:$src), "lslwhi\t$rd",
                         [(set i16:$rd, (AVRlslhi i16:$src)), (implicit SREG)]>;
 
   def LSLWNRd : Pseudo<(outs DLDREGS
                         : $rd),
                        (ins DREGS
                         : $src, imm16
                         : $bits),
                        "lslwn\t$rd, $bits", [
                          (set i16
                           : $rd, (AVRlslwn i16
                                   : $src, imm
                                   : $bits)),
                          (implicit SREG)
                        ]>;
 
   def LSLBNRd : Pseudo<(outs LD8
                         : $rd),
                        (ins GPR8
                         : $src, imm_ldi8
                         : $bits),
                        "lslbn\t$rd, $bits", [
                          (set i8
                           : $rd, (AVRlslbn i8
                                   : $src, imm
                                   : $bits)),
                          (implicit SREG)
                        ]>;
 
   def LSRRd
       : FRd<0b1001, 0b0100110,
             (outs GPR8
              : $rd),
             (ins GPR8
              : $src),
             "lsr\t$rd", [(set i8
                           : $rd, (AVRlsr i8
                                   : $src)),
                          (implicit SREG)]>;
 
   def LSRWRd : Pseudo<(outs DREGS
                        : $rd),
                       (ins DREGS
                        : $src),
                       "lsrw\t$rd",
                       [(set i16
                         : $rd, (AVRlsr i16
                                 : $src)),
                        (implicit SREG)]>;
 
   def LSRWLoRd : Pseudo<(outs DREGS:$rd), (ins DREGS:$src), "lsrwlo\t$rd",
                         [(set i16:$rd, (AVRlsrlo i16:$src)), (implicit SREG)]>;
 
   def LSRWNRd : Pseudo<(outs DLDREGS
                         : $rd),
                        (ins DREGS
                         : $src, imm16
                         : $bits),
                        "lsrwn\t$rd, $bits", [
                          (set i16
                           : $rd, (AVRlsrwn i16
                                   : $src, imm
                                   : $bits)),
                          (implicit SREG)
                        ]>;
 
   def LSRBNRd : Pseudo<(outs LD8
                         : $rd),
                        (ins GPR8
                         : $src, imm_ldi8
                         : $bits),
                        "lsrbn\t$rd, $bits", [
                          (set i8
                           : $rd, (AVRlsrbn i8
                                   : $src, imm
                                   : $bits)),
                          (implicit SREG)
                        ]>;
 
   def ASRRd
       : FRd<0b1001, 0b0100101,
             (outs GPR8
              : $rd),
             (ins GPR8
              : $src),
             "asr\t$rd", [(set i8
                           : $rd, (AVRasr i8
                                   : $src)),
                          (implicit SREG)]>;
 
   def ASRWNRd : Pseudo<(outs DLDREGS
                         : $rd),
                        (ins DREGS
                         : $src, imm16
                         : $bits),
                        "asrwn\t$rd, $bits", [
                          (set i16
                           : $rd, (AVRasrwn i16
                                   : $src, imm
                                   : $bits)),
                          (implicit SREG)
                        ]>;
 
   def ASRBNRd : Pseudo<(outs LD8
                         : $rd),
                        (ins GPR8
                         : $src, imm_ldi8
                         : $bits),
                        "asrbn\t$rd, $bits", [
                          (set i8
                           : $rd, (AVRasrbn i8
                                   : $src, imm
                                   : $bits)),
                          (implicit SREG)
                        ]>;
 
   def ASRWRd : Pseudo<(outs DREGS
                        : $rd),
                       (ins DREGS
                        : $src),
                       "asrw\t$rd",
                       [(set i16
                         : $rd, (AVRasr i16
                                 : $src)),
                        (implicit SREG)]>;
 
   def ASRWLoRd : Pseudo<(outs DREGS:$rd), (ins DREGS:$src), "asrwlo\t$rd",
                         [(set i16:$rd, (AVRasrlo i16:$src)), (implicit SREG)]>;
 
   def ROLBRd : Pseudo<(outs GPR8
                        : $rd),
                       (ins GPR8
                        : $src),
                       "rolb\t$rd",
                       [(set i8
                         : $rd, (AVRrol i8
                                 : $src)),
                        (implicit SREG)]>;
 
   def RORBRd : Pseudo<(outs GPR8
                        : $rd),
                       (ins GPR8
                        : $src),
                       "rorb\t$rd",
                       [(set i8
                         : $rd, (AVRror i8
                                 : $src)),
                        (implicit SREG)]>;
 
   // Bit rotate operations.
   let Uses = [SREG] in {
 
     def ROLWRd
         : Pseudo<(outs DREGS
                   : $rd),
                  (ins DREGS
                   : $src),
                  "rolw\t$rd",
                  [(set i16
                    : $rd, (AVRrol i16
                            : $src)),
                   (implicit SREG)]>;
 
     def RORRd : FRd<0b1001, 0b0100111,
                     (outs GPR8
                      : $rd),
                     (ins GPR8
                      : $src),
                     "ror\t$rd", []>;
 
     def RORWRd
         : Pseudo<(outs DREGS
                   : $rd),
                  (ins DREGS
                   : $src),
                  "rorw\t$rd",
                  [(set i16
                    : $rd, (AVRror i16
                            : $src)),
                   (implicit SREG)]>;
   }
 }
 
 // SWAP Rd
 // Swaps the high and low nibbles in a register.
 let Constraints =
     "$src = $rd" in def SWAPRd : FRd<0b1001, 0b0100010,
                                      (outs GPR8
                                       : $rd),
                                      (ins GPR8
                                       : $src),
                                      "swap\t$rd", [(set i8
                                                     : $rd, (AVRSwap i8
                                                             : $src))]>;
 
 // IO register bit set/clear operations.
 //: TODO: add patterns when popcount(imm)==2 to be expanded with 2 sbi/cbi
 // instead of in+ori+out which requires one more instr.
 def SBIAb : FIOBIT<0b10, (outs),
                    (ins imm_port5
                     : $addr, i8imm
                     : $bit),
                    "sbi\t$addr, $bit", [(store(or(i8(load lowioaddr8
                                                      : $addr)),
                                                iobitpos8
                                                : $bit),
                                          lowioaddr8
                                          : $addr)]>;
 
 def CBIAb : FIOBIT<0b00, (outs),
                    (ins imm_port5
                     : $addr, i8imm
                     : $bit),
                    "cbi\t$addr, $bit", [(store(and(i8(load lowioaddr8
                                                       : $addr)),
                                                iobitposn8
                                                : $bit),
                                          lowioaddr8
                                          : $addr)]>;
 
 // Status register bit load/store operations.
 let Defs = [SREG] in def BST : FRdB<0b01, (outs),
                                     (ins GPR8
                                      : $rd, i8imm
                                      : $b),
                                     "bst\t$rd, $b", []>;
 
 let Constraints = "$src = $rd",
     Uses = [SREG] in def BLD : FRdB<0b00,
                                     (outs GPR8
                                      : $rd),
                                     (ins GPR8
                                      : $src, i8imm
                                      : $b),
                                     "bld\t$rd, $b", []>;
 
 def CBR : InstAlias<"cbr\t$rd, $k", (ANDIRdK LD8 : $rd, imm_com8 : $k), 0>;
 
 // CLR Rd
 // Alias for EOR Rd, Rd
 // -------------
 // Clears all bits in a register.
 def CLR : InstAlias<"clr\t$rd", (EORRdRr GPR8 : $rd, GPR8 : $rd)>;
 
 // LSL Rd
 // Alias for ADD Rd, Rd
 // --------------
 // Logical shift left one bit.
 def LSL : InstAlias<"lsl\t$rd", (ADDRdRr GPR8 : $rd, GPR8 : $rd)>;
 
 def ROL : InstAlias<"rol\t$rd", (ADCRdRr GPR8 : $rd, GPR8 : $rd)>;
 
 // SER Rd
 // Alias for LDI Rd, 0xff
 // ---------
 // Sets all bits in a register.
 def : InstAlias<"ser\t$rd", (LDIRdK LD8 : $rd, 0xff), 0>;
 
 let Defs = [SREG] in def BSETs : FS<0, (outs),
                                     (ins i8imm
                                      : $s),
                                     "bset\t$s", []>;
 
 let Defs = [SREG] in def BCLRs : FS<1, (outs),
                                     (ins i8imm
                                      : $s),
                                     "bclr\t$s", []>;
 
 // Set/clear aliases for the carry (C) status flag (bit 0).
 def : InstAlias<"sec", (BSETs 0)>;
 def : InstAlias<"clc", (BCLRs 0)>;
 
 // Set/clear aliases for the zero (Z) status flag (bit 1).
 def : InstAlias<"sez", (BSETs 1)>;
 def : InstAlias<"clz", (BCLRs 1)>;
 
 // Set/clear aliases for the negative (N) status flag (bit 2).
 def : InstAlias<"sen", (BSETs 2)>;
 def : InstAlias<"cln", (BCLRs 2)>;
 
 // Set/clear aliases for the overflow (V) status flag (bit 3).
 def : InstAlias<"sev", (BSETs 3)>;
 def : InstAlias<"clv", (BCLRs 3)>;
 
 // Set/clear aliases for the signed (S) status flag (bit 4).
 def : InstAlias<"ses", (BSETs 4)>;
 def : InstAlias<"cls", (BCLRs 4)>;
 
 // Set/clear aliases for the half-carry (H) status flag (bit 5).
 def : InstAlias<"seh", (BSETs 5)>;
 def : InstAlias<"clh", (BCLRs 5)>;
 
 // Set/clear aliases for the T status flag (bit 6).
 def : InstAlias<"set", (BSETs 6)>;
 def : InstAlias<"clt", (BCLRs 6)>;
 
 // Set/clear aliases for the interrupt (I) status flag (bit 7).
 def : InstAlias<"sei", (BSETs 7)>;
 def : InstAlias<"cli", (BCLRs 7)>;
 
 //===----------------------------------------------------------------------===//
 // Special/Control instructions
 //===----------------------------------------------------------------------===//
 
 // BREAK
 // Breakpoint instruction
 // ---------
 // <|1001|0101|1001|1000>
 def BREAK : F16<0b1001010110011000, (outs), (ins), "break", []>,
             Requires<[HasBREAK]>;
 
 // NOP
 // No-operation instruction
 // ---------
 // <|0000|0000|0000|0000>
 def NOP : F16<0b0000000000000000, (outs), (ins), "nop", []>;
 
 // SLEEP
 // Sleep instruction
 // ---------
 // <|1001|0101|1000|1000>
 def SLEEP : F16<0b1001010110001000, (outs), (ins), "sleep", []>;
 
 // WDR
 // Watchdog reset
 // ---------
 // <|1001|0101|1010|1000>
 def WDR : F16<0b1001010110101000, (outs), (ins), "wdr", []>;
 
 //===----------------------------------------------------------------------===//
 // Pseudo instructions for later expansion
 //===----------------------------------------------------------------------===//
 
 //: TODO: Optimize this for wider types AND optimize the following code
 //       compile int foo(char a, char b, char c, char d) {return d+b;}
 //       looks like a missed sext_inreg opportunity.
 def SEXT
     : ExtensionPseudo<(outs DREGS
                        : $dst),
                       (ins GPR8
                        : $src),
                       "sext\t$dst, $src",
                       [(set i16
                         : $dst, (sext i8
                                  : $src)),
                        (implicit SREG)]>;
 
 def ZEXT
     : ExtensionPseudo<(outs DREGS
                        : $dst),
                       (ins GPR8
                        : $src),
                       "zext\t$dst, $src",
                       [(set i16
                         : $dst, (zext i8
                                  : $src)),
                        (implicit SREG)]>;
 
 // This pseudo gets expanded into a movw+adiw thus it clobbers SREG.
 let Defs = [SREG],
     hasSideEffects = 0 in def FRMIDX : Pseudo<(outs DLDREGS
                                                : $dst),
                                               (ins DLDREGS
                                                : $src, i16imm
                                                : $src2),
                                               "frmidx\t$dst, $src, $src2", []>;
 
 // This pseudo is either converted to a regular store or a push which clobbers
 // SP.
 def STDSPQRr : StorePseudo<(outs),
                            (ins memspi
                             : $dst, GPR8
                             : $src),
                            "stdstk\t$dst, $src", [(store i8
                                                    : $src, addr
                                                    : $dst)]>;
 
 // This pseudo is either converted to a regular store or a push which clobbers
 // SP.
 def STDWSPQRr : StorePseudo<(outs),
                             (ins memspi
                              : $dst, DREGS
                              : $src),
                             "stdwstk\t$dst, $src", [(store i16
                                                      : $src, addr
                                                      : $dst)]>;
 
 // SP read/write pseudos.
 let hasSideEffects = 0 in {
   let Uses = [SP] in def SPREAD : Pseudo<(outs DREGS
                                           : $dst),
                                          (ins GPRSP
                                           : $src),
                                          "spread\t$dst, $src", []>;
 
   let Defs = [SP] in def SPWRITE : Pseudo<(outs GPRSP
                                            : $dst),
                                           (ins DREGS
                                            : $src),
                                           "spwrite\t$dst, $src", []>;
 }
 
 def Select8 : SelectPseudo<(outs GPR8
                             : $dst),
                            (ins GPR8
                             : $src, GPR8
                             : $src2, i8imm
                             : $cc),
                            "# Select8 PSEUDO", [(set i8
                                                  : $dst, (AVRselectcc i8
                                                           : $src, i8
                                                           : $src2, imm
                                                           : $cc))]>;
 
 def Select16 : SelectPseudo<(outs DREGS
                              : $dst),
                             (ins DREGS
                              : $src, DREGS
                              : $src2, i8imm
                              : $cc),
                             "# Select16 PSEUDO", [(set i16
                                                    : $dst, (AVRselectcc i16
                                                             : $src, i16
                                                             : $src2, imm
                                                             : $cc))]>;
 
 def Lsl8 : ShiftPseudo<(outs GPR8
                         : $dst),
                        (ins GPR8
                         : $src, GPR8
                         : $cnt),
                        "# Lsl8 PSEUDO", [(set i8
                                           : $dst, (AVRlslLoop i8
                                                    : $src, i8
                                                    : $cnt))]>;
 
 def Lsl16 : ShiftPseudo<(outs DREGS
                          : $dst),
                         (ins DREGS
                          : $src, GPR8
                          : $cnt),
                         "# Lsl16 PSEUDO", [(set i16
                                             : $dst, (AVRlslLoop i16
                                                      : $src, i8
                                                      : $cnt))]>;
 
 def Lsr8 : ShiftPseudo<(outs GPR8
                         : $dst),
                        (ins GPR8
                         : $src, GPR8
                         : $cnt),
                        "# Lsr8 PSEUDO", [(set i8
                                           : $dst, (AVRlsrLoop i8
                                                    : $src, i8
                                                    : $cnt))]>;
 
 def Lsr16 : ShiftPseudo<(outs DREGS
                          : $dst),
                         (ins DREGS
                          : $src, GPR8
                          : $cnt),
                         "# Lsr16 PSEUDO", [(set i16
                                             : $dst, (AVRlsrLoop i16
                                                      : $src, i8
                                                      : $cnt))]>;
 
 def Rol8 : ShiftPseudo<(outs GPR8
                         : $dst),
                        (ins GPR8
                         : $src, GPR8
                         : $cnt),
                        "# Rol8 PSEUDO", [(set i8
                                           : $dst, (AVRrolLoop i8
                                                    : $src, i8
                                                    : $cnt))]>;
 
 def Rol16 : ShiftPseudo<(outs DREGS
                          : $dst),
                         (ins DREGS
                          : $src, GPR8
                          : $cnt),
                         "# Rol16 PSEUDO", [(set i16
                                             : $dst, (AVRrolLoop i16
                                                      : $src, i8
                                                      : $cnt))]>;
 
 def Ror8 : ShiftPseudo<(outs GPR8
                         : $dst),
                        (ins GPR8
                         : $src, GPR8
                         : $cnt),
                        "# Ror8 PSEUDO", [(set i8
                                           : $dst, (AVRrorLoop i8
                                                    : $src, i8
                                                    : $cnt))]>;
 
 def Ror16 : ShiftPseudo<(outs DREGS
                          : $dst),
                         (ins DREGS
                          : $src, GPR8
                          : $cnt),
                         "# Ror16 PSEUDO", [(set i16
                                             : $dst, (AVRrorLoop i16
                                                      : $src, i8
                                                      : $cnt))]>;
 
 def Asr8 : ShiftPseudo<(outs GPR8
                         : $dst),
                        (ins GPR8
                         : $src, GPR8
                         : $cnt),
                        "# Asr8 PSEUDO", [(set i8
                                           : $dst, (AVRasrLoop i8
                                                    : $src, i8
                                                    : $cnt))]>;
 
 def Asr16 : ShiftPseudo<(outs DREGS
                          : $dst),
                         (ins DREGS
                          : $src, GPR8
                          : $cnt),
                         "# Asr16 PSEUDO", [(set i16
                                             : $dst, (AVRasrLoop i16
                                                      : $src, i8
                                                      : $cnt))]>;
 
 // lowered to a copy from R1, which contains the value zero.
 let usesCustomInserter=1 in
 def CopyR1 : Pseudo<(outs GPR8:$rd), (ins), "clrz\t$rd", [(set i8:$rd, 0)]>;
 
 //===----------------------------------------------------------------------===//
 // Non-Instruction Patterns
 //===----------------------------------------------------------------------===//
 
 //: TODO: look in x86InstrCompiler.td for odd encoding trick related to
 // add x, 128 -> sub x, -128. Clang is emitting an eor for this (ldi+eor)
 
 // the add instruction always writes the carry flag
 def : Pat<(addc i8 : $src, i8 : $src2), (ADDRdRr i8 : $src, i8 : $src2)>;
 def : Pat<(addc DREGS
            : $src, DREGS
            : $src2),
           (ADDWRdRr DREGS
            : $src, DREGS
            : $src2)>;
 
 // all sub instruction variants always writes the carry flag
 def : Pat<(subc i8 : $src, i8 : $src2), (SUBRdRr i8 : $src, i8 : $src2)>;
 def : Pat<(subc i16 : $src, i16 : $src2), (SUBWRdRr i16 : $src, i16 : $src2)>;
 def : Pat<(subc i8 : $src, imm : $src2), (SUBIRdK i8 : $src, imm : $src2)>;
 def : Pat<(subc i16 : $src, imm : $src2), (SUBIWRdK i16 : $src, imm : $src2)>;
 
 // These patterns convert add (x, -imm) to sub (x, imm) since we dont have
 // any add with imm instructions. Also take care of the adiw/sbiw instructions.
 def : Pat<(add i16
            : $src1, imm0_63_neg
            : $src2),
           (SBIWRdK i16
            : $src1, (imm0_63_neg
                      : $src2))>;
 def : Pat<(add i16
            : $src1, imm
            : $src2),
           (SUBIWRdK i16
            : $src1, (imm16_neg_XFORM imm
                      : $src2))>;
 def : Pat<(addc i16
            : $src1, imm
            : $src2),
           (SUBIWRdK i16
            : $src1, (imm16_neg_XFORM imm
                      : $src2))>;
 
 def : Pat<(add i8
            : $src1, imm
            : $src2),
           (SUBIRdK i8
            : $src1, (imm8_neg_XFORM imm
                      : $src2))>;
 def : Pat<(addc i8
            : $src1, imm
            : $src2),
           (SUBIRdK i8
            : $src1, (imm8_neg_XFORM imm
                      : $src2))>;
 def : Pat<(adde i8
            : $src1, imm
            : $src2),
           (SBCIRdK i8
            : $src1, (imm8_neg_XFORM imm
                      : $src2))>;
 
 // Calls.
-def : Pat<(AVRcall(i16 tglobaladdr : $dst)), (CALLk tglobaladdr : $dst)>;
-def : Pat<(AVRcall(i16 texternalsym : $dst)), (CALLk texternalsym : $dst)>;
+let Predicates = [HasJMPCALL] in {
+  def : Pat<(AVRcall(i16 tglobaladdr:$dst)), (CALLk tglobaladdr:$dst)>;
+  def : Pat<(AVRcall(i16 texternalsym:$dst)), (CALLk texternalsym:$dst)>;
+}
+def : Pat<(AVRcall(i16 tglobaladdr:$dst)), (RCALLk tglobaladdr:$dst)>;
+def : Pat<(AVRcall(i16 texternalsym:$dst)), (RCALLk texternalsym:$dst)>;
 
 // `anyext`
 def : Pat<(i16(anyext i8
                : $src)),
           (INSERT_SUBREG(i16(IMPLICIT_DEF)), i8
            : $src, sub_lo)>;
 
 // `trunc`
 def : Pat<(i8(trunc i16 : $src)), (EXTRACT_SUBREG i16 : $src, sub_lo)>;
 
 // sext_inreg
 def : Pat<(sext_inreg i16
            : $src, i8),
           (SEXT(i8(EXTRACT_SUBREG i16
                    : $src, sub_lo)))>;
 
 // GlobalAddress
 def : Pat<(i16(AVRWrapper tglobaladdr : $dst)), (LDIWRdK tglobaladdr : $dst)>;
 def : Pat<(add i16
            : $src, (AVRWrapper tglobaladdr
                     : $src2)),
           (SUBIWRdK i16
            : $src, tglobaladdr
            : $src2)>;
 def : Pat<(i8(load(AVRWrapper tglobaladdr
                    : $dst))),
           (LDSRdK tglobaladdr
            : $dst)>;
 def : Pat<(i16(load(AVRWrapper tglobaladdr
                     : $dst))),
           (LDSWRdK tglobaladdr
            : $dst)>;
 def : Pat<(store i8
            : $src, (i16(AVRWrapper tglobaladdr
                         : $dst))),
           (STSKRr tglobaladdr
            : $dst, i8
            : $src)>;
 def : Pat<(store i16
            : $src, (i16(AVRWrapper tglobaladdr
                         : $dst))),
           (STSWKRr tglobaladdr
            : $dst, i16
            : $src)>;
 
 // BlockAddress
 def : Pat<(i16(AVRWrapper tblockaddress
                : $dst)),
           (LDIWRdK tblockaddress
            : $dst)>;
 
 def : Pat<(i8(trunc(AVRlsrwn DLDREGS
                     : $src, (i16 8)))),
           (EXTRACT_SUBREG DREGS
            : $src, sub_hi)>;
 
 // :FIXME: DAGCombiner produces an shl node after legalization from these seq:
 // BR_JT -> (mul x, 2) -> (shl x, 1)
 def : Pat<(shl i16 : $src1, (i8 1)), (LSLWRd i16 : $src1)>;
 
 // Lowering of 'tst' node to 'TST' instruction.
 // TST is an alias of AND Rd, Rd.
 def : Pat<(AVRtst i8 : $rd), (ANDRdRr GPR8 : $rd, GPR8 : $rd)>;
 
 // Lowering of 'lsl' node to 'LSL' instruction.
 // LSL is an alias of 'ADD Rd, Rd'
 def : Pat<(AVRlsl i8 : $rd), (ADDRdRr GPR8 : $rd, GPR8 : $rd)>;
diff --git a/contrib/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp b/contrib/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
index c83796b8579b..9eb546d1b5dc 100644
--- a/contrib/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
@@ -1,334 +1,339 @@
 //===-- SystemZMCCodeEmitter.cpp - Convert SystemZ code to machine code ---===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the SystemZMCCodeEmitter class.
 //
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/SystemZMCFixups.h"
 #include "MCTargetDesc/SystemZMCTargetDesc.h"
 #include "SystemZInstrInfo.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCFixup.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <cstdint>
 
 using namespace llvm;
 
 #define DEBUG_TYPE "mccodeemitter"
 
 namespace {
 
 class SystemZMCCodeEmitter : public MCCodeEmitter {
   const MCInstrInfo &MCII;
   MCContext &Ctx;
 
+  mutable unsigned MemOpsEmitted;
+
 public:
   SystemZMCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx)
     : MCII(mcii), Ctx(ctx) {
   }
 
   ~SystemZMCCodeEmitter() override = default;
 
   // OVerride MCCodeEmitter.
   void encodeInstruction(const MCInst &MI, raw_ostream &OS,
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const override;
 
 private:
   // Automatically generated by TableGen.
   uint64_t getBinaryCodeForInstr(const MCInst &MI,
                                  SmallVectorImpl<MCFixup> &Fixups,
                                  const MCSubtargetInfo &STI) const;
 
   // Called by the TableGen code to get the binary encoding of operand
   // MO in MI.  Fixups is the list of fixups against MI.
   uint64_t getMachineOpValue(const MCInst &MI, const MCOperand &MO,
                              SmallVectorImpl<MCFixup> &Fixups,
                              const MCSubtargetInfo &STI) const;
 
   // Return the displacement value for the OpNum operand. If it is a symbol,
   // add a fixup for it and return 0.
   uint64_t getDispOpValue(const MCInst &MI, unsigned OpNum,
                           SmallVectorImpl<MCFixup> &Fixups,
                           SystemZ::FixupKind Kind) const;
 
   // Called by the TableGen code to get the binary encoding of an address.
   // The index or length, if any, is encoded first, followed by the base,
   // followed by the displacement.  In a 20-bit displacement,
   // the low 12 bits are encoded before the high 8 bits.
   uint64_t getBDAddr12Encoding(const MCInst &MI, unsigned OpNum,
                                SmallVectorImpl<MCFixup> &Fixups,
                                const MCSubtargetInfo &STI) const;
   uint64_t getBDAddr20Encoding(const MCInst &MI, unsigned OpNum,
                                SmallVectorImpl<MCFixup> &Fixups,
                                const MCSubtargetInfo &STI) const;
   uint64_t getBDXAddr12Encoding(const MCInst &MI, unsigned OpNum,
                                 SmallVectorImpl<MCFixup> &Fixups,
                                 const MCSubtargetInfo &STI) const;
   uint64_t getBDXAddr20Encoding(const MCInst &MI, unsigned OpNum,
                                 SmallVectorImpl<MCFixup> &Fixups,
                                 const MCSubtargetInfo &STI) const;
   uint64_t getBDLAddr12Len4Encoding(const MCInst &MI, unsigned OpNum,
                                     SmallVectorImpl<MCFixup> &Fixups,
                                     const MCSubtargetInfo &STI) const;
   uint64_t getBDLAddr12Len8Encoding(const MCInst &MI, unsigned OpNum,
                                     SmallVectorImpl<MCFixup> &Fixups,
                                     const MCSubtargetInfo &STI) const;
   uint64_t getBDRAddr12Encoding(const MCInst &MI, unsigned OpNum,
                                 SmallVectorImpl<MCFixup> &Fixups,
                                 const MCSubtargetInfo &STI) const;
   uint64_t getBDVAddr12Encoding(const MCInst &MI, unsigned OpNum,
                                 SmallVectorImpl<MCFixup> &Fixups,
                                 const MCSubtargetInfo &STI) const;
 
   // Operand OpNum of MI needs a PC-relative fixup of kind Kind at
   // Offset bytes from the start of MI.  Add the fixup to Fixups
   // and return the in-place addend, which since we're a RELA target
   // is always 0.  If AllowTLS is true and optional operand OpNum + 1
   // is present, also emit a TLS call fixup for it.
   uint64_t getPCRelEncoding(const MCInst &MI, unsigned OpNum,
                             SmallVectorImpl<MCFixup> &Fixups,
                             unsigned Kind, int64_t Offset,
                             bool AllowTLS) const;
 
   uint64_t getPC16DBLEncoding(const MCInst &MI, unsigned OpNum,
                               SmallVectorImpl<MCFixup> &Fixups,
                               const MCSubtargetInfo &STI) const {
     return getPCRelEncoding(MI, OpNum, Fixups,
                             SystemZ::FK_390_PC16DBL, 2, false);
   }
   uint64_t getPC32DBLEncoding(const MCInst &MI, unsigned OpNum,
                               SmallVectorImpl<MCFixup> &Fixups,
                               const MCSubtargetInfo &STI) const {
     return getPCRelEncoding(MI, OpNum, Fixups,
                             SystemZ::FK_390_PC32DBL, 2, false);
   }
   uint64_t getPC16DBLTLSEncoding(const MCInst &MI, unsigned OpNum,
                                  SmallVectorImpl<MCFixup> &Fixups,
                                  const MCSubtargetInfo &STI) const {
     return getPCRelEncoding(MI, OpNum, Fixups,
                             SystemZ::FK_390_PC16DBL, 2, true);
   }
   uint64_t getPC32DBLTLSEncoding(const MCInst &MI, unsigned OpNum,
                                  SmallVectorImpl<MCFixup> &Fixups,
                                  const MCSubtargetInfo &STI) const {
     return getPCRelEncoding(MI, OpNum, Fixups,
                             SystemZ::FK_390_PC32DBL, 2, true);
   }
   uint64_t getPC12DBLBPPEncoding(const MCInst &MI, unsigned OpNum,
                                  SmallVectorImpl<MCFixup> &Fixups,
                                  const MCSubtargetInfo &STI) const {
     return getPCRelEncoding(MI, OpNum, Fixups,
                             SystemZ::FK_390_PC12DBL, 1, false);
   }
   uint64_t getPC16DBLBPPEncoding(const MCInst &MI, unsigned OpNum,
                                  SmallVectorImpl<MCFixup> &Fixups,
                                  const MCSubtargetInfo &STI) const {
     return getPCRelEncoding(MI, OpNum, Fixups,
                             SystemZ::FK_390_PC16DBL, 4, false);
   }
   uint64_t getPC24DBLBPPEncoding(const MCInst &MI, unsigned OpNum,
                                  SmallVectorImpl<MCFixup> &Fixups,
                                  const MCSubtargetInfo &STI) const {
     return getPCRelEncoding(MI, OpNum, Fixups,
                             SystemZ::FK_390_PC24DBL, 3, false);
   }
 
 private:
   FeatureBitset computeAvailableFeatures(const FeatureBitset &FB) const;
   void
   verifyInstructionPredicates(const MCInst &MI,
                               const FeatureBitset &AvailableFeatures) const;
 };
 
 } // end anonymous namespace
 
 void SystemZMCCodeEmitter::
 encodeInstruction(const MCInst &MI, raw_ostream &OS,
                   SmallVectorImpl<MCFixup> &Fixups,
                   const MCSubtargetInfo &STI) const {
   verifyInstructionPredicates(MI,
                               computeAvailableFeatures(STI.getFeatureBits()));
 
+  MemOpsEmitted = 0;
   uint64_t Bits = getBinaryCodeForInstr(MI, Fixups, STI);
   unsigned Size = MCII.get(MI.getOpcode()).getSize();
   // Big-endian insertion of Size bytes.
   unsigned ShiftValue = (Size * 8) - 8;
   for (unsigned I = 0; I != Size; ++I) {
     OS << uint8_t(Bits >> ShiftValue);
     ShiftValue -= 8;
   }
 }
 
 uint64_t SystemZMCCodeEmitter::
 getMachineOpValue(const MCInst &MI, const MCOperand &MO,
                   SmallVectorImpl<MCFixup> &Fixups,
                   const MCSubtargetInfo &STI) const {
   if (MO.isReg())
     return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg());
   if (MO.isImm())
     return static_cast<uint64_t>(MO.getImm());
   llvm_unreachable("Unexpected operand type!");
 }
 
 uint64_t SystemZMCCodeEmitter::
 getDispOpValue(const MCInst &MI, unsigned OpNum,
                SmallVectorImpl<MCFixup> &Fixups,
                SystemZ::FixupKind Kind) const {
   const MCOperand &MO = MI.getOperand(OpNum);
-  if (MO.isImm())
+  if (MO.isImm()) {
+    ++MemOpsEmitted;
     return static_cast<uint64_t>(MO.getImm());
+  }
   if (MO.isExpr()) {
     // All instructions follow the pattern where the first displacement has a
     // 2 bytes offset, and the second one 4 bytes.
-    unsigned ByteOffs = Fixups.size() == 0 ? 2 : 4;
+    unsigned ByteOffs = MemOpsEmitted++ == 0 ? 2 : 4;
     Fixups.push_back(MCFixup::create(ByteOffs, MO.getExpr(), (MCFixupKind)Kind,
                                      MI.getLoc()));
     assert(Fixups.size() <= 2 && "More than two memory operands in MI?");
     return 0;
   }
   llvm_unreachable("Unexpected operand type!");
 }
 
 uint64_t SystemZMCCodeEmitter::
 getBDAddr12Encoding(const MCInst &MI, unsigned OpNum,
                     SmallVectorImpl<MCFixup> &Fixups,
                     const MCSubtargetInfo &STI) const {
   uint64_t Base = getMachineOpValue(MI, MI.getOperand(OpNum), Fixups, STI);
   uint64_t Disp = getDispOpValue(MI, OpNum + 1, Fixups, SystemZ::FK_390_12);
   assert(isUInt<4>(Base) && isUInt<12>(Disp));
   return (Base << 12) | Disp;
 }
 
 uint64_t SystemZMCCodeEmitter::
 getBDAddr20Encoding(const MCInst &MI, unsigned OpNum,
                     SmallVectorImpl<MCFixup> &Fixups,
                     const MCSubtargetInfo &STI) const {
   uint64_t Base = getMachineOpValue(MI, MI.getOperand(OpNum), Fixups, STI);
   uint64_t Disp = getDispOpValue(MI, OpNum + 1, Fixups, SystemZ::FK_390_20);
   assert(isUInt<4>(Base) && isInt<20>(Disp));
   return (Base << 20) | ((Disp & 0xfff) << 8) | ((Disp & 0xff000) >> 12);
 }
 
 uint64_t SystemZMCCodeEmitter::
 getBDXAddr12Encoding(const MCInst &MI, unsigned OpNum,
                      SmallVectorImpl<MCFixup> &Fixups,
                      const MCSubtargetInfo &STI) const {
   uint64_t Base = getMachineOpValue(MI, MI.getOperand(OpNum), Fixups, STI);
   uint64_t Disp = getDispOpValue(MI, OpNum + 1, Fixups, SystemZ::FK_390_12);
   uint64_t Index = getMachineOpValue(MI, MI.getOperand(OpNum + 2), Fixups, STI);
   assert(isUInt<4>(Base) && isUInt<12>(Disp) && isUInt<4>(Index));
   return (Index << 16) | (Base << 12) | Disp;
 }
 
 uint64_t SystemZMCCodeEmitter::
 getBDXAddr20Encoding(const MCInst &MI, unsigned OpNum,
                      SmallVectorImpl<MCFixup> &Fixups,
                      const MCSubtargetInfo &STI) const {
   uint64_t Base = getMachineOpValue(MI, MI.getOperand(OpNum), Fixups, STI);
   uint64_t Disp = getDispOpValue(MI, OpNum + 1, Fixups, SystemZ::FK_390_20);
   uint64_t Index = getMachineOpValue(MI, MI.getOperand(OpNum + 2), Fixups, STI);
   assert(isUInt<4>(Base) && isInt<20>(Disp) && isUInt<4>(Index));
   return (Index << 24) | (Base << 20) | ((Disp & 0xfff) << 8)
     | ((Disp & 0xff000) >> 12);
 }
 
 uint64_t SystemZMCCodeEmitter::
 getBDLAddr12Len4Encoding(const MCInst &MI, unsigned OpNum,
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const {
   uint64_t Base = getMachineOpValue(MI, MI.getOperand(OpNum), Fixups, STI);
   uint64_t Disp = getDispOpValue(MI, OpNum + 1, Fixups, SystemZ::FK_390_12);
   uint64_t Len  = getMachineOpValue(MI, MI.getOperand(OpNum + 2), Fixups, STI) - 1;
   assert(isUInt<4>(Base) && isUInt<12>(Disp) && isUInt<4>(Len));
   return (Len << 16) | (Base << 12) | Disp;
 }
 
 uint64_t SystemZMCCodeEmitter::
 getBDLAddr12Len8Encoding(const MCInst &MI, unsigned OpNum,
                          SmallVectorImpl<MCFixup> &Fixups,
                          const MCSubtargetInfo &STI) const {
   uint64_t Base = getMachineOpValue(MI, MI.getOperand(OpNum), Fixups, STI);
   uint64_t Disp = getDispOpValue(MI, OpNum + 1, Fixups, SystemZ::FK_390_12);
   uint64_t Len  = getMachineOpValue(MI, MI.getOperand(OpNum + 2), Fixups, STI) - 1;
   assert(isUInt<4>(Base) && isUInt<12>(Disp) && isUInt<8>(Len));
   return (Len << 16) | (Base << 12) | Disp;
 }
 
 uint64_t SystemZMCCodeEmitter::
 getBDRAddr12Encoding(const MCInst &MI, unsigned OpNum,
                      SmallVectorImpl<MCFixup> &Fixups,
                      const MCSubtargetInfo &STI) const {
   uint64_t Base = getMachineOpValue(MI, MI.getOperand(OpNum), Fixups, STI);
   uint64_t Disp = getDispOpValue(MI, OpNum + 1, Fixups, SystemZ::FK_390_12);
   uint64_t Len  = getMachineOpValue(MI, MI.getOperand(OpNum + 2), Fixups, STI);
   assert(isUInt<4>(Base) && isUInt<12>(Disp) && isUInt<4>(Len));
   return (Len << 16) | (Base << 12) | Disp;
 }
 
 uint64_t SystemZMCCodeEmitter::
 getBDVAddr12Encoding(const MCInst &MI, unsigned OpNum,
                      SmallVectorImpl<MCFixup> &Fixups,
                      const MCSubtargetInfo &STI) const {
   uint64_t Base = getMachineOpValue(MI, MI.getOperand(OpNum), Fixups, STI);
   uint64_t Disp = getDispOpValue(MI, OpNum + 1, Fixups, SystemZ::FK_390_12);
   uint64_t Index = getMachineOpValue(MI, MI.getOperand(OpNum + 2), Fixups, STI);
   assert(isUInt<4>(Base) && isUInt<12>(Disp) && isUInt<5>(Index));
   return (Index << 16) | (Base << 12) | Disp;
 }
 
 uint64_t
 SystemZMCCodeEmitter::getPCRelEncoding(const MCInst &MI, unsigned OpNum,
                                        SmallVectorImpl<MCFixup> &Fixups,
                                        unsigned Kind, int64_t Offset,
                                        bool AllowTLS) const {
   SMLoc Loc = MI.getLoc();
   const MCOperand &MO = MI.getOperand(OpNum);
   const MCExpr *Expr;
   if (MO.isImm())
     Expr = MCConstantExpr::create(MO.getImm() + Offset, Ctx);
   else {
     Expr = MO.getExpr();
     if (Offset) {
       // The operand value is relative to the start of MI, but the fixup
       // is relative to the operand field itself, which is Offset bytes
       // into MI.  Add Offset to the relocation value to cancel out
       // this difference.
       const MCExpr *OffsetExpr = MCConstantExpr::create(Offset, Ctx);
       Expr = MCBinaryExpr::createAdd(Expr, OffsetExpr, Ctx);
     }
   }
   Fixups.push_back(MCFixup::create(Offset, Expr, (MCFixupKind)Kind, Loc));
 
   // Output the fixup for the TLS marker if present.
   if (AllowTLS && OpNum + 1 < MI.getNumOperands()) {
     const MCOperand &MOTLS = MI.getOperand(OpNum + 1);
     Fixups.push_back(MCFixup::create(
         0, MOTLS.getExpr(), (MCFixupKind)SystemZ::FK_390_TLS_CALL, Loc));
   }
   return 0;
 }
 
 #define ENABLE_INSTR_PREDICATE_VERIFIER
 #include "SystemZGenMCCodeEmitter.inc"
 
 MCCodeEmitter *llvm::createSystemZMCCodeEmitter(const MCInstrInfo &MCII,
                                                 const MCRegisterInfo &MRI,
                                                 MCContext &Ctx) {
   return new SystemZMCCodeEmitter(MCII, Ctx);
 }
diff --git a/contrib/llvm-project/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/contrib/llvm-project/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index f10651d5c5d7..d795697b3ed6 100644
--- a/contrib/llvm-project/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/contrib/llvm-project/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -1,8736 +1,8740 @@
 //===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the SystemZTargetLowering class.
 //
 //===----------------------------------------------------------------------===//
 
 #include "SystemZISelLowering.h"
 #include "SystemZCallingConv.h"
 #include "SystemZConstantPoolValue.h"
 #include "SystemZMachineFunctionInfo.h"
 #include "SystemZTargetMachine.h"
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsS390.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/KnownBits.h"
 #include <cctype>
 
 using namespace llvm;
 
 #define DEBUG_TYPE "systemz-lower"
 
 namespace {
 // Represents information about a comparison.
 struct Comparison {
   Comparison(SDValue Op0In, SDValue Op1In, SDValue ChainIn)
     : Op0(Op0In), Op1(Op1In), Chain(ChainIn),
       Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
 
   // The operands to the comparison.
   SDValue Op0, Op1;
 
   // Chain if this is a strict floating-point comparison.
   SDValue Chain;
 
   // The opcode that should be used to compare Op0 and Op1.
   unsigned Opcode;
 
   // A SystemZICMP value.  Only used for integer comparisons.
   unsigned ICmpType;
 
   // The mask of CC values that Opcode can produce.
   unsigned CCValid;
 
   // The mask of CC values for which the original condition is true.
   unsigned CCMask;
 };
 } // end anonymous namespace
 
 // Classify VT as either 32 or 64 bit.
 static bool is32Bit(EVT VT) {
   switch (VT.getSimpleVT().SimpleTy) {
   case MVT::i32:
     return true;
   case MVT::i64:
     return false;
   default:
     llvm_unreachable("Unsupported type");
   }
 }
 
 // Return a version of MachineOperand that can be safely used before the
 // final use.
 static MachineOperand earlyUseOperand(MachineOperand Op) {
   if (Op.isReg())
     Op.setIsKill(false);
   return Op;
 }
 
 SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
                                              const SystemZSubtarget &STI)
     : TargetLowering(TM), Subtarget(STI) {
   MVT PtrVT = MVT::getIntegerVT(8 * TM.getPointerSize(0));
 
   auto *Regs = STI.getSpecialRegisters();
 
   // Set up the register classes.
   if (Subtarget.hasHighWord())
     addRegisterClass(MVT::i32, &SystemZ::GRX32BitRegClass);
   else
     addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass);
   addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass);
   if (!useSoftFloat()) {
     if (Subtarget.hasVector()) {
       addRegisterClass(MVT::f32, &SystemZ::VR32BitRegClass);
       addRegisterClass(MVT::f64, &SystemZ::VR64BitRegClass);
     } else {
       addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass);
       addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass);
     }
     if (Subtarget.hasVectorEnhancements1())
       addRegisterClass(MVT::f128, &SystemZ::VR128BitRegClass);
     else
       addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass);
 
     if (Subtarget.hasVector()) {
       addRegisterClass(MVT::v16i8, &SystemZ::VR128BitRegClass);
       addRegisterClass(MVT::v8i16, &SystemZ::VR128BitRegClass);
       addRegisterClass(MVT::v4i32, &SystemZ::VR128BitRegClass);
       addRegisterClass(MVT::v2i64, &SystemZ::VR128BitRegClass);
       addRegisterClass(MVT::v4f32, &SystemZ::VR128BitRegClass);
       addRegisterClass(MVT::v2f64, &SystemZ::VR128BitRegClass);
     }
   }
 
   // Compute derived properties from the register classes
   computeRegisterProperties(Subtarget.getRegisterInfo());
 
   // Set up special registers.
   setStackPointerRegisterToSaveRestore(Regs->getStackPointerRegister());
 
   // TODO: It may be better to default to latency-oriented scheduling, however
   // LLVM's current latency-oriented scheduler can't handle physreg definitions
   // such as SystemZ has with CC, so set this to the register-pressure
   // scheduler, because it can.
   setSchedulingPreference(Sched::RegPressure);
 
   setBooleanContents(ZeroOrOneBooleanContent);
   setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
   // Instructions are strings of 2-byte aligned 2-byte values.
   setMinFunctionAlignment(Align(2));
   // For performance reasons we prefer 16-byte alignment.
   setPrefFunctionAlignment(Align(16));
 
   // Handle operations that are handled in a similar way for all types.
   for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
        I <= MVT::LAST_FP_VALUETYPE;
        ++I) {
     MVT VT = MVT::SimpleValueType(I);
     if (isTypeLegal(VT)) {
       // Lower SET_CC into an IPM-based sequence.
       setOperationAction(ISD::SETCC, VT, Custom);
       setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
       setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
 
       // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE).
       setOperationAction(ISD::SELECT, VT, Expand);
 
       // Lower SELECT_CC and BR_CC into separate comparisons and branches.
       setOperationAction(ISD::SELECT_CC, VT, Custom);
       setOperationAction(ISD::BR_CC,     VT, Custom);
     }
   }
 
   // Expand jump table branches as address arithmetic followed by an
   // indirect jump.
   setOperationAction(ISD::BR_JT, MVT::Other, Expand);
 
   // Expand BRCOND into a BR_CC (see above).
   setOperationAction(ISD::BRCOND, MVT::Other, Expand);
 
   // Handle integer types.
   for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
        I <= MVT::LAST_INTEGER_VALUETYPE;
        ++I) {
     MVT VT = MVT::SimpleValueType(I);
     if (isTypeLegal(VT)) {
       setOperationAction(ISD::ABS, VT, Legal);
 
       // Expand individual DIV and REMs into DIVREMs.
       setOperationAction(ISD::SDIV, VT, Expand);
       setOperationAction(ISD::UDIV, VT, Expand);
       setOperationAction(ISD::SREM, VT, Expand);
       setOperationAction(ISD::UREM, VT, Expand);
       setOperationAction(ISD::SDIVREM, VT, Custom);
       setOperationAction(ISD::UDIVREM, VT, Custom);
 
       // Support addition/subtraction with overflow.
       setOperationAction(ISD::SADDO, VT, Custom);
       setOperationAction(ISD::SSUBO, VT, Custom);
 
       // Support addition/subtraction with carry.
       setOperationAction(ISD::UADDO, VT, Custom);
       setOperationAction(ISD::USUBO, VT, Custom);
 
       // Support carry in as value rather than glue.
       setOperationAction(ISD::ADDCARRY, VT, Custom);
       setOperationAction(ISD::SUBCARRY, VT, Custom);
 
       // Lower ATOMIC_LOAD and ATOMIC_STORE into normal volatile loads and
       // stores, putting a serialization instruction after the stores.
       setOperationAction(ISD::ATOMIC_LOAD,  VT, Custom);
       setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
 
       // Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are
       // available, or if the operand is constant.
       setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
 
       // Use POPCNT on z196 and above.
       if (Subtarget.hasPopulationCount())
         setOperationAction(ISD::CTPOP, VT, Custom);
       else
         setOperationAction(ISD::CTPOP, VT, Expand);
 
       // No special instructions for these.
       setOperationAction(ISD::CTTZ,            VT, Expand);
       setOperationAction(ISD::ROTR,            VT, Expand);
 
       // Use *MUL_LOHI where possible instead of MULH*.
       setOperationAction(ISD::MULHS, VT, Expand);
       setOperationAction(ISD::MULHU, VT, Expand);
       setOperationAction(ISD::SMUL_LOHI, VT, Custom);
       setOperationAction(ISD::UMUL_LOHI, VT, Custom);
 
       // Only z196 and above have native support for conversions to unsigned.
       // On z10, promoting to i64 doesn't generate an inexact condition for
       // values that are outside the i32 range but in the i64 range, so use
       // the default expansion.
       if (!Subtarget.hasFPExtension())
         setOperationAction(ISD::FP_TO_UINT, VT, Expand);
 
       // Mirror those settings for STRICT_FP_TO_[SU]INT.  Note that these all
       // default to Expand, so need to be modified to Legal where appropriate.
       setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Legal);
       if (Subtarget.hasFPExtension())
         setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Legal);
 
       // And similarly for STRICT_[SU]INT_TO_FP.
       setOperationAction(ISD::STRICT_SINT_TO_FP, VT, Legal);
       if (Subtarget.hasFPExtension())
         setOperationAction(ISD::STRICT_UINT_TO_FP, VT, Legal);
     }
   }
 
   // Type legalization will convert 8- and 16-bit atomic operations into
   // forms that operate on i32s (but still keeping the original memory VT).
   // Lower them into full i32 operations.
   setOperationAction(ISD::ATOMIC_SWAP,      MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_ADD,  MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_SUB,  MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_AND,  MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_OR,   MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_XOR,  MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_MIN,  MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_MAX,  MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom);
 
   // Even though i128 is not a legal type, we still need to custom lower
   // the atomic operations in order to exploit SystemZ instructions.
   setOperationAction(ISD::ATOMIC_LOAD,     MVT::i128, Custom);
   setOperationAction(ISD::ATOMIC_STORE,    MVT::i128, Custom);
 
   // We can use the CC result of compare-and-swap to implement
   // the "success" result of ATOMIC_CMP_SWAP_WITH_SUCCESS.
   setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i32, Custom);
   setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i64, Custom);
   setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
 
   setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
 
   // Traps are legal, as we will convert them to "j .+2".
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
 
   // z10 has instructions for signed but not unsigned FP conversion.
   // Handle unsigned 32-bit types as signed 64-bit types.
   if (!Subtarget.hasFPExtension()) {
     setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote);
     setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand);
     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Promote);
     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Expand);
   }
 
   // We have native support for a 64-bit CTLZ, via FLOGR.
   setOperationAction(ISD::CTLZ, MVT::i32, Promote);
   setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Promote);
   setOperationAction(ISD::CTLZ, MVT::i64, Legal);
 
   // On z15 we have native support for a 64-bit CTPOP.
   if (Subtarget.hasMiscellaneousExtensions3()) {
     setOperationAction(ISD::CTPOP, MVT::i32, Promote);
     setOperationAction(ISD::CTPOP, MVT::i64, Legal);
   }
 
   // Give LowerOperation the chance to replace 64-bit ORs with subregs.
   setOperationAction(ISD::OR, MVT::i64, Custom);
 
   // Expand 128 bit shifts without using a libcall.
   setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand);
   setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand);
   setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand);
   setLibcallName(RTLIB::SRL_I128, nullptr);
   setLibcallName(RTLIB::SHL_I128, nullptr);
   setLibcallName(RTLIB::SRA_I128, nullptr);
 
   // Handle bitcast from fp128 to i128.
   setOperationAction(ISD::BITCAST, MVT::i128, Custom);
 
   // We have native instructions for i8, i16 and i32 extensions, but not i1.
   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
   for (MVT VT : MVT::integer_valuetypes()) {
     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
     setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote);
     setLoadExtAction(ISD::EXTLOAD,  VT, MVT::i1, Promote);
   }
 
   // Handle the various types of symbolic address.
   setOperationAction(ISD::ConstantPool,     PtrVT, Custom);
   setOperationAction(ISD::GlobalAddress,    PtrVT, Custom);
   setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom);
   setOperationAction(ISD::BlockAddress,     PtrVT, Custom);
   setOperationAction(ISD::JumpTable,        PtrVT, Custom);
 
   // We need to handle dynamic allocations specially because of the
   // 160-byte area at the bottom of the stack.
   setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom);
   setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, PtrVT, Custom);
 
   setOperationAction(ISD::STACKSAVE,    MVT::Other, Custom);
   setOperationAction(ISD::STACKRESTORE, MVT::Other, Custom);
 
   // Handle prefetches with PFD or PFDRL.
   setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
 
   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
     // Assume by default that all vector operations need to be expanded.
     for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode)
       if (getOperationAction(Opcode, VT) == Legal)
         setOperationAction(Opcode, VT, Expand);
 
     // Likewise all truncating stores and extending loads.
     for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
       setTruncStoreAction(VT, InnerVT, Expand);
       setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
       setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
       setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
     }
 
     if (isTypeLegal(VT)) {
       // These operations are legal for anything that can be stored in a
       // vector register, even if there is no native support for the format
       // as such.  In particular, we can do these for v4f32 even though there
       // are no specific instructions for that format.
       setOperationAction(ISD::LOAD, VT, Legal);
       setOperationAction(ISD::STORE, VT, Legal);
       setOperationAction(ISD::VSELECT, VT, Legal);
       setOperationAction(ISD::BITCAST, VT, Legal);
       setOperationAction(ISD::UNDEF, VT, Legal);
 
       // Likewise, except that we need to replace the nodes with something
       // more specific.
       setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
       setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
     }
   }
 
   // Handle integer vector types.
   for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
     if (isTypeLegal(VT)) {
       // These operations have direct equivalents.
       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Legal);
       setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Legal);
       setOperationAction(ISD::ADD, VT, Legal);
       setOperationAction(ISD::SUB, VT, Legal);
       if (VT != MVT::v2i64)
         setOperationAction(ISD::MUL, VT, Legal);
       setOperationAction(ISD::ABS, VT, Legal);
       setOperationAction(ISD::AND, VT, Legal);
       setOperationAction(ISD::OR, VT, Legal);
       setOperationAction(ISD::XOR, VT, Legal);
       if (Subtarget.hasVectorEnhancements1())
         setOperationAction(ISD::CTPOP, VT, Legal);
       else
         setOperationAction(ISD::CTPOP, VT, Custom);
       setOperationAction(ISD::CTTZ, VT, Legal);
       setOperationAction(ISD::CTLZ, VT, Legal);
 
       // Convert a GPR scalar to a vector by inserting it into element 0.
       setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
 
       // Use a series of unpacks for extensions.
       setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
       setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
 
       // Detect shifts by a scalar amount and convert them into
       // V*_BY_SCALAR.
       setOperationAction(ISD::SHL, VT, Custom);
       setOperationAction(ISD::SRA, VT, Custom);
       setOperationAction(ISD::SRL, VT, Custom);
 
       // At present ROTL isn't matched by DAGCombiner.  ROTR should be
       // converted into ROTL.
       setOperationAction(ISD::ROTL, VT, Expand);
       setOperationAction(ISD::ROTR, VT, Expand);
 
       // Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
       // and inverting the result as necessary.
       setOperationAction(ISD::SETCC, VT, Custom);
       setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
       if (Subtarget.hasVectorEnhancements1())
         setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
     }
   }
 
   if (Subtarget.hasVector()) {
     // There should be no need to check for float types other than v2f64
     // since <2 x f32> isn't a legal type.
     setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal);
     setOperationAction(ISD::FP_TO_SINT, MVT::v2f64, Legal);
     setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal);
     setOperationAction(ISD::FP_TO_UINT, MVT::v2f64, Legal);
     setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal);
     setOperationAction(ISD::SINT_TO_FP, MVT::v2f64, Legal);
     setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal);
     setOperationAction(ISD::UINT_TO_FP, MVT::v2f64, Legal);
 
     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i64, Legal);
     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i64, Legal);
     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i64, Legal);
     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i64, Legal);
     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f64, Legal);
   }
 
   if (Subtarget.hasVectorEnhancements2()) {
     setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal);
     setOperationAction(ISD::FP_TO_SINT, MVT::v4f32, Legal);
     setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal);
     setOperationAction(ISD::FP_TO_UINT, MVT::v4f32, Legal);
     setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal);
     setOperationAction(ISD::SINT_TO_FP, MVT::v4f32, Legal);
     setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal);
     setOperationAction(ISD::UINT_TO_FP, MVT::v4f32, Legal);
 
     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4i32, Legal);
     setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4f32, Legal);
     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4i32, Legal);
     setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4f32, Legal);
     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i32, Legal);
     setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4f32, Legal);
     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32, Legal);
     setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4f32, Legal);
   }
 
   // Handle floating-point types.
   for (unsigned I = MVT::FIRST_FP_VALUETYPE;
        I <= MVT::LAST_FP_VALUETYPE;
        ++I) {
     MVT VT = MVT::SimpleValueType(I);
     if (isTypeLegal(VT)) {
       // We can use FI for FRINT.
       setOperationAction(ISD::FRINT, VT, Legal);
 
       // We can use the extended form of FI for other rounding operations.
       if (Subtarget.hasFPExtension()) {
         setOperationAction(ISD::FNEARBYINT, VT, Legal);
         setOperationAction(ISD::FFLOOR, VT, Legal);
         setOperationAction(ISD::FCEIL, VT, Legal);
         setOperationAction(ISD::FTRUNC, VT, Legal);
         setOperationAction(ISD::FROUND, VT, Legal);
       }
 
       // No special instructions for these.
       setOperationAction(ISD::FSIN, VT, Expand);
       setOperationAction(ISD::FCOS, VT, Expand);
       setOperationAction(ISD::FSINCOS, VT, Expand);
       setOperationAction(ISD::FREM, VT, Expand);
       setOperationAction(ISD::FPOW, VT, Expand);
 
       // Handle constrained floating-point operations.
       setOperationAction(ISD::STRICT_FADD, VT, Legal);
       setOperationAction(ISD::STRICT_FSUB, VT, Legal);
       setOperationAction(ISD::STRICT_FMUL, VT, Legal);
       setOperationAction(ISD::STRICT_FDIV, VT, Legal);
       setOperationAction(ISD::STRICT_FMA, VT, Legal);
       setOperationAction(ISD::STRICT_FSQRT, VT, Legal);
       setOperationAction(ISD::STRICT_FRINT, VT, Legal);
       setOperationAction(ISD::STRICT_FP_ROUND, VT, Legal);
       setOperationAction(ISD::STRICT_FP_EXTEND, VT, Legal);
       if (Subtarget.hasFPExtension()) {
         setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
         setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
         setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
         setOperationAction(ISD::STRICT_FROUND, VT, Legal);
         setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
       }
     }
   }
 
   // Handle floating-point vector types.
   if (Subtarget.hasVector()) {
     // Scalar-to-vector conversion is just a subreg.
     setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal);
     setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal);
 
     // Some insertions and extractions can be done directly but others
     // need to go via integers.
     setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
     setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom);
     setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
     setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom);
 
     // These operations have direct equivalents.
     setOperationAction(ISD::FADD, MVT::v2f64, Legal);
     setOperationAction(ISD::FNEG, MVT::v2f64, Legal);
     setOperationAction(ISD::FSUB, MVT::v2f64, Legal);
     setOperationAction(ISD::FMUL, MVT::v2f64, Legal);
     setOperationAction(ISD::FMA, MVT::v2f64, Legal);
     setOperationAction(ISD::FDIV, MVT::v2f64, Legal);
     setOperationAction(ISD::FABS, MVT::v2f64, Legal);
     setOperationAction(ISD::FSQRT, MVT::v2f64, Legal);
     setOperationAction(ISD::FRINT, MVT::v2f64, Legal);
     setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal);
     setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal);
     setOperationAction(ISD::FCEIL, MVT::v2f64, Legal);
     setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal);
     setOperationAction(ISD::FROUND, MVT::v2f64, Legal);
 
     // Handle constrained floating-point operations.
     setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_FMUL, MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_FMA, MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_FRINT, MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_FNEARBYINT, MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_FFLOOR, MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_FCEIL, MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_FTRUNC, MVT::v2f64, Legal);
     setOperationAction(ISD::STRICT_FROUND, MVT::v2f64, Legal);
   }
 
   // The vector enhancements facility 1 has instructions for these.
   if (Subtarget.hasVectorEnhancements1()) {
     setOperationAction(ISD::FADD, MVT::v4f32, Legal);
     setOperationAction(ISD::FNEG, MVT::v4f32, Legal);
     setOperationAction(ISD::FSUB, MVT::v4f32, Legal);
     setOperationAction(ISD::FMUL, MVT::v4f32, Legal);
     setOperationAction(ISD::FMA, MVT::v4f32, Legal);
     setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
     setOperationAction(ISD::FABS, MVT::v4f32, Legal);
     setOperationAction(ISD::FSQRT, MVT::v4f32, Legal);
     setOperationAction(ISD::FRINT, MVT::v4f32, Legal);
     setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal);
     setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal);
     setOperationAction(ISD::FCEIL, MVT::v4f32, Legal);
     setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal);
     setOperationAction(ISD::FROUND, MVT::v4f32, Legal);
 
     setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
     setOperationAction(ISD::FMAXIMUM, MVT::f64, Legal);
     setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
     setOperationAction(ISD::FMINIMUM, MVT::f64, Legal);
 
     setOperationAction(ISD::FMAXNUM, MVT::v2f64, Legal);
     setOperationAction(ISD::FMAXIMUM, MVT::v2f64, Legal);
     setOperationAction(ISD::FMINNUM, MVT::v2f64, Legal);
     setOperationAction(ISD::FMINIMUM, MVT::v2f64, Legal);
 
     setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
     setOperationAction(ISD::FMAXIMUM, MVT::f32, Legal);
     setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
     setOperationAction(ISD::FMINIMUM, MVT::f32, Legal);
 
     setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal);
     setOperationAction(ISD::FMAXIMUM, MVT::v4f32, Legal);
     setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal);
     setOperationAction(ISD::FMINIMUM, MVT::v4f32, Legal);
 
     setOperationAction(ISD::FMAXNUM, MVT::f128, Legal);
     setOperationAction(ISD::FMAXIMUM, MVT::f128, Legal);
     setOperationAction(ISD::FMINNUM, MVT::f128, Legal);
     setOperationAction(ISD::FMINIMUM, MVT::f128, Legal);
 
     // Handle constrained floating-point operations.
     setOperationAction(ISD::STRICT_FADD, MVT::v4f32, Legal);
     setOperationAction(ISD::STRICT_FSUB, MVT::v4f32, Legal);
     setOperationAction(ISD::STRICT_FMUL, MVT::v4f32, Legal);
     setOperationAction(ISD::STRICT_FMA, MVT::v4f32, Legal);
     setOperationAction(ISD::STRICT_FDIV, MVT::v4f32, Legal);
     setOperationAction(ISD::STRICT_FSQRT, MVT::v4f32, Legal);
     setOperationAction(ISD::STRICT_FRINT, MVT::v4f32, Legal);
     setOperationAction(ISD::STRICT_FNEARBYINT, MVT::v4f32, Legal);
     setOperationAction(ISD::STRICT_FFLOOR, MVT::v4f32, Legal);
     setOperationAction(ISD::STRICT_FCEIL, MVT::v4f32, Legal);
     setOperationAction(ISD::STRICT_FROUND, MVT::v4f32, Legal);
     setOperationAction(ISD::STRICT_FTRUNC, MVT::v4f32, Legal);
     for (auto VT : { MVT::f32, MVT::f64, MVT::f128,
                      MVT::v4f32, MVT::v2f64 }) {
       setOperationAction(ISD::STRICT_FMAXNUM, VT, Legal);
       setOperationAction(ISD::STRICT_FMINNUM, VT, Legal);
       setOperationAction(ISD::STRICT_FMAXIMUM, VT, Legal);
       setOperationAction(ISD::STRICT_FMINIMUM, VT, Legal);
     }
   }
 
   // We only have fused f128 multiply-addition on vector registers.
   if (!Subtarget.hasVectorEnhancements1()) {
     setOperationAction(ISD::FMA, MVT::f128, Expand);
     setOperationAction(ISD::STRICT_FMA, MVT::f128, Expand);
   }
 
   // We don't have a copysign instruction on vector registers.
   if (Subtarget.hasVectorEnhancements1())
     setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand);
 
   // Needed so that we don't try to implement f128 constant loads using
   // a load-and-extend of a f80 constant (in cases where the constant
   // would fit in an f80).
   for (MVT VT : MVT::fp_valuetypes())
     setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand);
 
   // We don't have extending load instruction on vector registers.
   if (Subtarget.hasVectorEnhancements1()) {
     setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f32, Expand);
     setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f64, Expand);
   }
 
   // Floating-point truncation and stores need to be done separately.
   setTruncStoreAction(MVT::f64,  MVT::f32, Expand);
   setTruncStoreAction(MVT::f128, MVT::f32, Expand);
   setTruncStoreAction(MVT::f128, MVT::f64, Expand);
 
   // We have 64-bit FPR<->GPR moves, but need special handling for
   // 32-bit forms.
   if (!Subtarget.hasVector()) {
     setOperationAction(ISD::BITCAST, MVT::i32, Custom);
     setOperationAction(ISD::BITCAST, MVT::f32, Custom);
   }
 
   // VASTART and VACOPY need to deal with the SystemZ-specific varargs
   // structure, but VAEND is a no-op.
   setOperationAction(ISD::VASTART, MVT::Other, Custom);
   setOperationAction(ISD::VACOPY,  MVT::Other, Custom);
   setOperationAction(ISD::VAEND,   MVT::Other, Expand);
 
   // Codes for which we want to perform some z-specific combinations.
   setTargetDAGCombine(ISD::ZERO_EXTEND);
   setTargetDAGCombine(ISD::SIGN_EXTEND);
   setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
   setTargetDAGCombine(ISD::LOAD);
   setTargetDAGCombine(ISD::STORE);
   setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
   setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
   setTargetDAGCombine(ISD::FP_ROUND);
   setTargetDAGCombine(ISD::STRICT_FP_ROUND);
   setTargetDAGCombine(ISD::FP_EXTEND);
   setTargetDAGCombine(ISD::SINT_TO_FP);
   setTargetDAGCombine(ISD::UINT_TO_FP);
   setTargetDAGCombine(ISD::STRICT_FP_EXTEND);
   setTargetDAGCombine(ISD::BSWAP);
   setTargetDAGCombine(ISD::SDIV);
   setTargetDAGCombine(ISD::UDIV);
   setTargetDAGCombine(ISD::SREM);
   setTargetDAGCombine(ISD::UREM);
   setTargetDAGCombine(ISD::INTRINSIC_VOID);
   setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
 
   // Handle intrinsics.
   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
 
   // We want to use MVC in preference to even a single load/store pair.
   MaxStoresPerMemcpy = 0;
   MaxStoresPerMemcpyOptSize = 0;
 
   // The main memset sequence is a byte store followed by an MVC.
   // Two STC or MV..I stores win over that, but the kind of fused stores
   // generated by target-independent code don't when the byte value is
   // variable.  E.g.  "STC <reg>;MHI <reg>,257;STH <reg>" is not better
   // than "STC;MVC".  Handle the choice in target-specific code instead.
   MaxStoresPerMemset = 0;
   MaxStoresPerMemsetOptSize = 0;
 
   // Default to having -disable-strictnode-mutation on
   IsStrictFPEnabled = true;
 }
 
 bool SystemZTargetLowering::useSoftFloat() const {
   return Subtarget.hasSoftFloat();
 }
 
 EVT SystemZTargetLowering::getSetCCResultType(const DataLayout &DL,
                                               LLVMContext &, EVT VT) const {
   if (!VT.isVector())
     return MVT::i32;
   return VT.changeVectorElementTypeToInteger();
 }
 
 bool SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(
     const MachineFunction &MF, EVT VT) const {
   VT = VT.getScalarType();
 
   if (!VT.isSimple())
     return false;
 
   switch (VT.getSimpleVT().SimpleTy) {
   case MVT::f32:
   case MVT::f64:
     return true;
   case MVT::f128:
     return Subtarget.hasVectorEnhancements1();
   default:
     break;
   }
 
   return false;
 }
 
 // Return true if the constant can be generated with a vector instruction,
 // such as VGM, VGMB or VREPI.
 bool SystemZVectorConstantInfo::isVectorConstantLegal(
     const SystemZSubtarget &Subtarget) {
   const SystemZInstrInfo *TII =
       static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
   if (!Subtarget.hasVector() ||
       (isFP128 && !Subtarget.hasVectorEnhancements1()))
     return false;
 
   // Try using VECTOR GENERATE BYTE MASK.  This is the architecturally-
   // preferred way of creating all-zero and all-one vectors so give it
   // priority over other methods below.
   unsigned Mask = 0;
   unsigned I = 0;
   for (; I < SystemZ::VectorBytes; ++I) {
     uint64_t Byte = IntBits.lshr(I * 8).trunc(8).getZExtValue();
     if (Byte == 0xff)
       Mask |= 1ULL << I;
     else if (Byte != 0)
       break;
   }
   if (I == SystemZ::VectorBytes) {
     Opcode = SystemZISD::BYTE_MASK;
     OpVals.push_back(Mask);
     VecVT = MVT::getVectorVT(MVT::getIntegerVT(8), 16);
     return true;
   }
 
   if (SplatBitSize > 64)
     return false;
 
   auto tryValue = [&](uint64_t Value) -> bool {
     // Try VECTOR REPLICATE IMMEDIATE
     int64_t SignedValue = SignExtend64(Value, SplatBitSize);
     if (isInt<16>(SignedValue)) {
       OpVals.push_back(((unsigned) SignedValue));
       Opcode = SystemZISD::REPLICATE;
       VecVT = MVT::getVectorVT(MVT::getIntegerVT(SplatBitSize),
                                SystemZ::VectorBits / SplatBitSize);
       return true;
     }
     // Try VECTOR GENERATE MASK
     unsigned Start, End;
     if (TII->isRxSBGMask(Value, SplatBitSize, Start, End)) {
       // isRxSBGMask returns the bit numbers for a full 64-bit value, with 0
       // denoting 1 << 63 and 63 denoting 1.  Convert them to bit numbers for
       // an SplatBitSize value, so that 0 denotes 1 << (SplatBitSize-1).
       OpVals.push_back(Start - (64 - SplatBitSize));
       OpVals.push_back(End - (64 - SplatBitSize));
       Opcode = SystemZISD::ROTATE_MASK;
       VecVT = MVT::getVectorVT(MVT::getIntegerVT(SplatBitSize),
                                SystemZ::VectorBits / SplatBitSize);
       return true;
     }
     return false;
   };
 
   // First try assuming that any undefined bits above the highest set bit
   // and below the lowest set bit are 1s.  This increases the likelihood of
   // being able to use a sign-extended element value in VECTOR REPLICATE
   // IMMEDIATE or a wraparound mask in VECTOR GENERATE MASK.
   uint64_t SplatBitsZ = SplatBits.getZExtValue();
   uint64_t SplatUndefZ = SplatUndef.getZExtValue();
   uint64_t Lower =
       (SplatUndefZ & ((uint64_t(1) << findFirstSet(SplatBitsZ)) - 1));
   uint64_t Upper =
       (SplatUndefZ & ~((uint64_t(1) << findLastSet(SplatBitsZ)) - 1));
   if (tryValue(SplatBitsZ | Upper | Lower))
     return true;
 
   // Now try assuming that any undefined bits between the first and
   // last defined set bits are set.  This increases the chances of
   // using a non-wraparound mask.
   uint64_t Middle = SplatUndefZ & ~Upper & ~Lower;
   return tryValue(SplatBitsZ | Middle);
 }
 
 SystemZVectorConstantInfo::SystemZVectorConstantInfo(APFloat FPImm) {
   IntBits = FPImm.bitcastToAPInt().zextOrSelf(128);
   isFP128 = (&FPImm.getSemantics() == &APFloat::IEEEquad());
   SplatBits = FPImm.bitcastToAPInt();
   unsigned Width = SplatBits.getBitWidth();
   IntBits <<= (SystemZ::VectorBits - Width);
 
   // Find the smallest splat.
   while (Width > 8) {
     unsigned HalfSize = Width / 2;
     APInt HighValue = SplatBits.lshr(HalfSize).trunc(HalfSize);
     APInt LowValue = SplatBits.trunc(HalfSize);
 
     // If the two halves do not match, stop here.
     if (HighValue != LowValue || 8 > HalfSize)
       break;
 
     SplatBits = HighValue;
     Width = HalfSize;
   }
   SplatUndef = 0;
   SplatBitSize = Width;
 }
 
 SystemZVectorConstantInfo::SystemZVectorConstantInfo(BuildVectorSDNode *BVN) {
   assert(BVN->isConstant() && "Expected a constant BUILD_VECTOR");
   bool HasAnyUndefs;
 
   // Get IntBits by finding the 128 bit splat.
   BVN->isConstantSplat(IntBits, SplatUndef, SplatBitSize, HasAnyUndefs, 128,
                        true);
 
   // Get SplatBits by finding the 8 bit or greater splat.
   BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, 8,
                        true);
 }
 
 bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
                                          bool ForCodeSize) const {
   // We can load zero using LZ?R and negative zero using LZ?R;LC?BR.
   if (Imm.isZero() || Imm.isNegZero())
     return true;
 
   return SystemZVectorConstantInfo(Imm).isVectorConstantLegal(Subtarget);
 }
 
 /// Returns true if stack probing through inline assembly is requested.
 bool SystemZTargetLowering::hasInlineStackProbe(MachineFunction &MF) const {
   // If the function specifically requests inline stack probes, emit them.
   if (MF.getFunction().hasFnAttribute("probe-stack"))
     return MF.getFunction().getFnAttribute("probe-stack").getValueAsString() ==
            "inline-asm";
   return false;
 }
 
 bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
   // We can use CGFI or CLGFI.
   return isInt<32>(Imm) || isUInt<32>(Imm);
 }
 
 bool SystemZTargetLowering::isLegalAddImmediate(int64_t Imm) const {
   // We can use ALGFI or SLGFI.
   return isUInt<32>(Imm) || isUInt<32>(-Imm);
 }
 
 bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(
     EVT VT, unsigned, Align, MachineMemOperand::Flags, bool *Fast) const {
   // Unaligned accesses should never be slower than the expanded version.
   // We check specifically for aligned accesses in the few cases where
   // they are required.
   if (Fast)
     *Fast = true;
   return true;
 }
 
 // Information about the addressing mode for a memory access.
 struct AddressingMode {
   // True if a long displacement is supported.
   bool LongDisplacement;
 
   // True if use of index register is supported.
   bool IndexReg;
 
   AddressingMode(bool LongDispl, bool IdxReg) :
     LongDisplacement(LongDispl), IndexReg(IdxReg) {}
 };
 
 // Return the desired addressing mode for a Load which has only one use (in
 // the same block) which is a Store.
 static AddressingMode getLoadStoreAddrMode(bool HasVector,
                                           Type *Ty) {
   // With vector support a Load->Store combination may be combined to either
   // an MVC or vector operations and it seems to work best to allow the
   // vector addressing mode.
   if (HasVector)
     return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
 
   // Otherwise only the MVC case is special.
   bool MVC = Ty->isIntegerTy(8);
   return AddressingMode(!MVC/*LongDispl*/, !MVC/*IdxReg*/);
 }
 
 // Return the addressing mode which seems most desirable given an LLVM
 // Instruction pointer.
 static AddressingMode
 supportedAddressingMode(Instruction *I, bool HasVector) {
   if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
     switch (II->getIntrinsicID()) {
     default: break;
     case Intrinsic::memset:
     case Intrinsic::memmove:
     case Intrinsic::memcpy:
       return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
     }
   }
 
   if (isa<LoadInst>(I) && I->hasOneUse()) {
     auto *SingleUser = cast<Instruction>(*I->user_begin());
     if (SingleUser->getParent() == I->getParent()) {
       if (isa<ICmpInst>(SingleUser)) {
         if (auto *C = dyn_cast<ConstantInt>(SingleUser->getOperand(1)))
           if (C->getBitWidth() <= 64 &&
               (isInt<16>(C->getSExtValue()) || isUInt<16>(C->getZExtValue())))
             // Comparison of memory with 16 bit signed / unsigned immediate
             return AddressingMode(false/*LongDispl*/, false/*IdxReg*/);
       } else if (isa<StoreInst>(SingleUser))
         // Load->Store
         return getLoadStoreAddrMode(HasVector, I->getType());
     }
   } else if (auto *StoreI = dyn_cast<StoreInst>(I)) {
     if (auto *LoadI = dyn_cast<LoadInst>(StoreI->getValueOperand()))
       if (LoadI->hasOneUse() && LoadI->getParent() == I->getParent())
         // Load->Store
         return getLoadStoreAddrMode(HasVector, LoadI->getType());
   }
 
   if (HasVector && (isa<LoadInst>(I) || isa<StoreInst>(I))) {
 
     // * Use LDE instead of LE/LEY for z13 to avoid partial register
     //   dependencies (LDE only supports small offsets).
     // * Utilize the vector registers to hold floating point
     //   values (vector load / store instructions only support small
     //   offsets).
 
     Type *MemAccessTy = (isa<LoadInst>(I) ? I->getType() :
                          I->getOperand(0)->getType());
     bool IsFPAccess = MemAccessTy->isFloatingPointTy();
     bool IsVectorAccess = MemAccessTy->isVectorTy();
 
     // A store of an extracted vector element will be combined into a VSTE type
     // instruction.
     if (!IsVectorAccess && isa<StoreInst>(I)) {
       Value *DataOp = I->getOperand(0);
       if (isa<ExtractElementInst>(DataOp))
         IsVectorAccess = true;
     }
 
     // A load which gets inserted into a vector element will be combined into a
     // VLE type instruction.
     if (!IsVectorAccess && isa<LoadInst>(I) && I->hasOneUse()) {
       User *LoadUser = *I->user_begin();
       if (isa<InsertElementInst>(LoadUser))
         IsVectorAccess = true;
     }
 
     if (IsFPAccess || IsVectorAccess)
       return AddressingMode(false/*LongDispl*/, true/*IdxReg*/);
   }
 
   return AddressingMode(true/*LongDispl*/, true/*IdxReg*/);
 }
 
 bool SystemZTargetLowering::isLegalAddressingMode(const DataLayout &DL,
        const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I) const {
   // Punt on globals for now, although they can be used in limited
   // RELATIVE LONG cases.
   if (AM.BaseGV)
     return false;
 
   // Require a 20-bit signed offset.
   if (!isInt<20>(AM.BaseOffs))
     return false;
 
   AddressingMode SupportedAM(true, true);
   if (I != nullptr)
     SupportedAM = supportedAddressingMode(I, Subtarget.hasVector());
 
   if (!SupportedAM.LongDisplacement && !isUInt<12>(AM.BaseOffs))
     return false;
 
   if (!SupportedAM.IndexReg)
     // No indexing allowed.
     return AM.Scale == 0;
   else
     // Indexing is OK but no scale factor can be applied.
     return AM.Scale == 0 || AM.Scale == 1;
 }
 
 bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const {
   if (!FromType->isIntegerTy() || !ToType->isIntegerTy())
     return false;
   unsigned FromBits = FromType->getPrimitiveSizeInBits().getFixedSize();
   unsigned ToBits = ToType->getPrimitiveSizeInBits().getFixedSize();
   return FromBits > ToBits;
 }
 
 bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const {
   if (!FromVT.isInteger() || !ToVT.isInteger())
     return false;
   unsigned FromBits = FromVT.getFixedSizeInBits();
   unsigned ToBits = ToVT.getFixedSizeInBits();
   return FromBits > ToBits;
 }
 
 //===----------------------------------------------------------------------===//
 // Inline asm support
 //===----------------------------------------------------------------------===//
 
 TargetLowering::ConstraintType
 SystemZTargetLowering::getConstraintType(StringRef Constraint) const {
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
     case 'a': // Address register
     case 'd': // Data register (equivalent to 'r')
     case 'f': // Floating-point register
     case 'h': // High-part register
     case 'r': // General-purpose register
     case 'v': // Vector register
       return C_RegisterClass;
 
     case 'Q': // Memory with base and unsigned 12-bit displacement
     case 'R': // Likewise, plus an index
     case 'S': // Memory with base and signed 20-bit displacement
     case 'T': // Likewise, plus an index
     case 'm': // Equivalent to 'T'.
       return C_Memory;
 
     case 'I': // Unsigned 8-bit constant
     case 'J': // Unsigned 12-bit constant
     case 'K': // Signed 16-bit constant
     case 'L': // Signed 20-bit displacement (on all targets we support)
     case 'M': // 0x7fffffff
       return C_Immediate;
 
     default:
       break;
     }
   }
   return TargetLowering::getConstraintType(Constraint);
 }
 
 TargetLowering::ConstraintWeight SystemZTargetLowering::
 getSingleConstraintMatchWeight(AsmOperandInfo &info,
                                const char *constraint) const {
   ConstraintWeight weight = CW_Invalid;
   Value *CallOperandVal = info.CallOperandVal;
   // If we don't have a value, we can't do a match,
   // but allow it at the lowest weight.
   if (!CallOperandVal)
     return CW_Default;
   Type *type = CallOperandVal->getType();
   // Look at the constraint type.
   switch (*constraint) {
   default:
     weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
     break;
 
   case 'a': // Address register
   case 'd': // Data register (equivalent to 'r')
   case 'h': // High-part register
   case 'r': // General-purpose register
     if (CallOperandVal->getType()->isIntegerTy())
       weight = CW_Register;
     break;
 
   case 'f': // Floating-point register
     if (type->isFloatingPointTy())
       weight = CW_Register;
     break;
 
   case 'v': // Vector register
     if ((type->isVectorTy() || type->isFloatingPointTy()) &&
         Subtarget.hasVector())
       weight = CW_Register;
     break;
 
   case 'I': // Unsigned 8-bit constant
     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
       if (isUInt<8>(C->getZExtValue()))
         weight = CW_Constant;
     break;
 
   case 'J': // Unsigned 12-bit constant
     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
       if (isUInt<12>(C->getZExtValue()))
         weight = CW_Constant;
     break;
 
   case 'K': // Signed 16-bit constant
     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
       if (isInt<16>(C->getSExtValue()))
         weight = CW_Constant;
     break;
 
   case 'L': // Signed 20-bit displacement (on all targets we support)
     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
       if (isInt<20>(C->getSExtValue()))
         weight = CW_Constant;
     break;
 
   case 'M': // 0x7fffffff
     if (auto *C = dyn_cast<ConstantInt>(CallOperandVal))
       if (C->getZExtValue() == 0x7fffffff)
         weight = CW_Constant;
     break;
   }
   return weight;
 }
 
 // Parse a "{tNNN}" register constraint for which the register type "t"
 // has already been verified.  MC is the class associated with "t" and
 // Map maps 0-based register numbers to LLVM register numbers.
 static std::pair<unsigned, const TargetRegisterClass *>
 parseRegisterNumber(StringRef Constraint, const TargetRegisterClass *RC,
                     const unsigned *Map, unsigned Size) {
   assert(*(Constraint.end()-1) == '}' && "Missing '}'");
   if (isdigit(Constraint[2])) {
     unsigned Index;
     bool Failed =
         Constraint.slice(2, Constraint.size() - 1).getAsInteger(10, Index);
     if (!Failed && Index < Size && Map[Index])
       return std::make_pair(Map[Index], RC);
   }
   return std::make_pair(0U, nullptr);
 }
 
 std::pair<unsigned, const TargetRegisterClass *>
 SystemZTargetLowering::getRegForInlineAsmConstraint(
     const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
   if (Constraint.size() == 1) {
     // GCC Constraint Letters
     switch (Constraint[0]) {
     default: break;
     case 'd': // Data register (equivalent to 'r')
     case 'r': // General-purpose register
       if (VT == MVT::i64)
         return std::make_pair(0U, &SystemZ::GR64BitRegClass);
       else if (VT == MVT::i128)
         return std::make_pair(0U, &SystemZ::GR128BitRegClass);
       return std::make_pair(0U, &SystemZ::GR32BitRegClass);
 
     case 'a': // Address register
       if (VT == MVT::i64)
         return std::make_pair(0U, &SystemZ::ADDR64BitRegClass);
       else if (VT == MVT::i128)
         return std::make_pair(0U, &SystemZ::ADDR128BitRegClass);
       return std::make_pair(0U, &SystemZ::ADDR32BitRegClass);
 
     case 'h': // High-part register (an LLVM extension)
       return std::make_pair(0U, &SystemZ::GRH32BitRegClass);
 
     case 'f': // Floating-point register
       if (!useSoftFloat()) {
         if (VT == MVT::f64)
           return std::make_pair(0U, &SystemZ::FP64BitRegClass);
         else if (VT == MVT::f128)
           return std::make_pair(0U, &SystemZ::FP128BitRegClass);
         return std::make_pair(0U, &SystemZ::FP32BitRegClass);
       }
       break;
     case 'v': // Vector register
       if (Subtarget.hasVector()) {
         if (VT == MVT::f32)
           return std::make_pair(0U, &SystemZ::VR32BitRegClass);
         if (VT == MVT::f64)
           return std::make_pair(0U, &SystemZ::VR64BitRegClass);
         return std::make_pair(0U, &SystemZ::VR128BitRegClass);
       }
       break;
     }
   }
   if (Constraint.size() > 0 && Constraint[0] == '{') {
     // We need to override the default register parsing for GPRs and FPRs
     // because the interpretation depends on VT.  The internal names of
     // the registers are also different from the external names
     // (F0D and F0S instead of F0, etc.).
     if (Constraint[1] == 'r') {
       if (VT == MVT::i32)
         return parseRegisterNumber(Constraint, &SystemZ::GR32BitRegClass,
                                    SystemZMC::GR32Regs, 16);
       if (VT == MVT::i128)
         return parseRegisterNumber(Constraint, &SystemZ::GR128BitRegClass,
                                    SystemZMC::GR128Regs, 16);
       return parseRegisterNumber(Constraint, &SystemZ::GR64BitRegClass,
                                  SystemZMC::GR64Regs, 16);
     }
     if (Constraint[1] == 'f') {
       if (useSoftFloat())
         return std::make_pair(
             0u, static_cast<const TargetRegisterClass *>(nullptr));
       if (VT == MVT::f32)
         return parseRegisterNumber(Constraint, &SystemZ::FP32BitRegClass,
                                    SystemZMC::FP32Regs, 16);
       if (VT == MVT::f128)
         return parseRegisterNumber(Constraint, &SystemZ::FP128BitRegClass,
                                    SystemZMC::FP128Regs, 16);
       return parseRegisterNumber(Constraint, &SystemZ::FP64BitRegClass,
                                  SystemZMC::FP64Regs, 16);
     }
     if (Constraint[1] == 'v') {
       if (!Subtarget.hasVector())
         return std::make_pair(
             0u, static_cast<const TargetRegisterClass *>(nullptr));
       if (VT == MVT::f32)
         return parseRegisterNumber(Constraint, &SystemZ::VR32BitRegClass,
                                    SystemZMC::VR32Regs, 32);
       if (VT == MVT::f64)
         return parseRegisterNumber(Constraint, &SystemZ::VR64BitRegClass,
                                    SystemZMC::VR64Regs, 32);
       return parseRegisterNumber(Constraint, &SystemZ::VR128BitRegClass,
                                  SystemZMC::VR128Regs, 32);
     }
   }
   return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
 }
 
 // FIXME? Maybe this could be a TableGen attribute on some registers and
 // this table could be generated automatically from RegInfo.
 Register SystemZTargetLowering::getRegisterByName(const char *RegName, LLT VT,
                                                   const MachineFunction &MF) const {
 
   Register Reg = StringSwitch<Register>(RegName)
                    .Case("r15", SystemZ::R15D)
                    .Default(0);
   if (Reg)
     return Reg;
   report_fatal_error("Invalid register name global variable");
 }
 
 void SystemZTargetLowering::
 LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
                              std::vector<SDValue> &Ops,
                              SelectionDAG &DAG) const {
   // Only support length 1 constraints for now.
   if (Constraint.length() == 1) {
     switch (Constraint[0]) {
     case 'I': // Unsigned 8-bit constant
       if (auto *C = dyn_cast<ConstantSDNode>(Op))
         if (isUInt<8>(C->getZExtValue()))
           Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
                                               Op.getValueType()));
       return;
 
     case 'J': // Unsigned 12-bit constant
       if (auto *C = dyn_cast<ConstantSDNode>(Op))
         if (isUInt<12>(C->getZExtValue()))
           Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
                                               Op.getValueType()));
       return;
 
     case 'K': // Signed 16-bit constant
       if (auto *C = dyn_cast<ConstantSDNode>(Op))
         if (isInt<16>(C->getSExtValue()))
           Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
                                               Op.getValueType()));
       return;
 
     case 'L': // Signed 20-bit displacement (on all targets we support)
       if (auto *C = dyn_cast<ConstantSDNode>(Op))
         if (isInt<20>(C->getSExtValue()))
           Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
                                               Op.getValueType()));
       return;
 
     case 'M': // 0x7fffffff
       if (auto *C = dyn_cast<ConstantSDNode>(Op))
         if (C->getZExtValue() == 0x7fffffff)
           Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
                                               Op.getValueType()));
       return;
     }
   }
   TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
 }
 
 //===----------------------------------------------------------------------===//
 // Calling conventions
 //===----------------------------------------------------------------------===//
 
 #include "SystemZGenCallingConv.inc"
 
 const MCPhysReg *SystemZTargetLowering::getScratchRegisters(
   CallingConv::ID) const {
   static const MCPhysReg ScratchRegs[] = { SystemZ::R0D, SystemZ::R1D,
                                            SystemZ::R14D, 0 };
   return ScratchRegs;
 }
 
 bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType,
                                                      Type *ToType) const {
   return isTruncateFree(FromType, ToType);
 }
 
 bool SystemZTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
   return CI->isTailCall();
 }
 
 // We do not yet support 128-bit single-element vector types.  If the user
 // attempts to use such types as function argument or return type, prefer
 // to error out instead of emitting code violating the ABI.
 static void VerifyVectorType(MVT VT, EVT ArgVT) {
   if (ArgVT.isVector() && !VT.isVector())
     report_fatal_error("Unsupported vector argument or return type");
 }
 
 static void VerifyVectorTypes(const SmallVectorImpl<ISD::InputArg> &Ins) {
   for (unsigned i = 0; i < Ins.size(); ++i)
     VerifyVectorType(Ins[i].VT, Ins[i].ArgVT);
 }
 
 static void VerifyVectorTypes(const SmallVectorImpl<ISD::OutputArg> &Outs) {
   for (unsigned i = 0; i < Outs.size(); ++i)
     VerifyVectorType(Outs[i].VT, Outs[i].ArgVT);
 }
 
 // Value is a value that has been passed to us in the location described by VA
 // (and so has type VA.getLocVT()).  Convert Value to VA.getValVT(), chaining
 // any loads onto Chain.
 static SDValue convertLocVTToValVT(SelectionDAG &DAG, const SDLoc &DL,
                                    CCValAssign &VA, SDValue Chain,
                                    SDValue Value) {
   // If the argument has been promoted from a smaller type, insert an
   // assertion to capture this.
   if (VA.getLocInfo() == CCValAssign::SExt)
     Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value,
                         DAG.getValueType(VA.getValVT()));
   else if (VA.getLocInfo() == CCValAssign::ZExt)
     Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value,
                         DAG.getValueType(VA.getValVT()));
 
   if (VA.isExtInLoc())
     Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value);
   else if (VA.getLocInfo() == CCValAssign::BCvt) {
     // If this is a short vector argument loaded from the stack,
     // extend from i64 to full vector size and then bitcast.
     assert(VA.getLocVT() == MVT::i64);
     assert(VA.getValVT().isVector());
     Value = DAG.getBuildVector(MVT::v2i64, DL, {Value, DAG.getUNDEF(MVT::i64)});
     Value = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Value);
   } else
     assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo");
   return Value;
 }
 
 // Value is a value of type VA.getValVT() that we need to copy into
 // the location described by VA.  Return a copy of Value converted to
 // VA.getValVT().  The caller is responsible for handling indirect values.
 static SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL,
                                    CCValAssign &VA, SDValue Value) {
   switch (VA.getLocInfo()) {
   case CCValAssign::SExt:
     return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value);
   case CCValAssign::ZExt:
     return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value);
   case CCValAssign::AExt:
     return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value);
   case CCValAssign::BCvt: {
     assert(VA.getLocVT() == MVT::i64 || VA.getLocVT() == MVT::i128);
     assert(VA.getValVT().isVector() || VA.getValVT() == MVT::f64 ||
            VA.getValVT() == MVT::f128);
     MVT BitCastToType = VA.getValVT().isVector() && VA.getLocVT() == MVT::i64
                             ? MVT::v2i64
                             : VA.getLocVT();
     Value = DAG.getNode(ISD::BITCAST, DL, BitCastToType, Value);
     // For ELF, this is a short vector argument to be stored to the stack,
     // bitcast to v2i64 and then extract first element.
     if (BitCastToType == MVT::v2i64)
       return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VA.getLocVT(), Value,
                          DAG.getConstant(0, DL, MVT::i32));
     return Value;
   }
   case CCValAssign::Full:
     return Value;
   default:
     llvm_unreachable("Unhandled getLocInfo()");
   }
 }
 
 static SDValue lowerI128ToGR128(SelectionDAG &DAG, SDValue In) {
   SDLoc DL(In);
   SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, In,
                            DAG.getIntPtrConstant(0, DL));
   SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, In,
                            DAG.getIntPtrConstant(1, DL));
   SDNode *Pair = DAG.getMachineNode(SystemZ::PAIR128, DL,
                                     MVT::Untyped, Hi, Lo);
   return SDValue(Pair, 0);
 }
 
 static SDValue lowerGR128ToI128(SelectionDAG &DAG, SDValue In) {
   SDLoc DL(In);
   SDValue Hi = DAG.getTargetExtractSubreg(SystemZ::subreg_h64,
                                           DL, MVT::i64, In);
   SDValue Lo = DAG.getTargetExtractSubreg(SystemZ::subreg_l64,
                                           DL, MVT::i64, In);
   return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128, Lo, Hi);
 }
 
 bool SystemZTargetLowering::splitValueIntoRegisterParts(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
     unsigned NumParts, MVT PartVT, Optional<CallingConv::ID> CC) const {
   EVT ValueVT = Val.getValueType();
   assert((ValueVT != MVT::i128 ||
           ((NumParts == 1 && PartVT == MVT::Untyped) ||
            (NumParts == 2 && PartVT == MVT::i64))) &&
          "Unknown handling of i128 value.");
   if (ValueVT == MVT::i128 && NumParts == 1) {
     // Inline assembly operand.
     Parts[0] = lowerI128ToGR128(DAG, Val);
     return true;
   }
   return false;
 }
 
 SDValue SystemZTargetLowering::joinRegisterPartsIntoValue(
     SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
     MVT PartVT, EVT ValueVT, Optional<CallingConv::ID> CC) const {
   assert((ValueVT != MVT::i128 ||
           ((NumParts == 1 && PartVT == MVT::Untyped) ||
            (NumParts == 2 && PartVT == MVT::i64))) &&
          "Unknown handling of i128 value.");
   if (ValueVT == MVT::i128 && NumParts == 1)
     // Inline assembly operand.
     return lowerGR128ToI128(DAG, Parts[0]);
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::LowerFormalArguments(
     SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MachineRegisterInfo &MRI = MF.getRegInfo();
   SystemZMachineFunctionInfo *FuncInfo =
       MF.getInfo<SystemZMachineFunctionInfo>();
   auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
   // Detect unsupported vector argument types.
   if (Subtarget.hasVector())
     VerifyVectorTypes(Ins);
 
   // Assign locations to all of the incoming arguments.
   SmallVector<CCValAssign, 16> ArgLocs;
   SystemZCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
   CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ);
 
   unsigned NumFixedGPRs = 0;
   unsigned NumFixedFPRs = 0;
   for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
     SDValue ArgValue;
     CCValAssign &VA = ArgLocs[I];
     EVT LocVT = VA.getLocVT();
     if (VA.isRegLoc()) {
       // Arguments passed in registers
       const TargetRegisterClass *RC;
       switch (LocVT.getSimpleVT().SimpleTy) {
       default:
         // Integers smaller than i64 should be promoted to i64.
         llvm_unreachable("Unexpected argument type");
       case MVT::i32:
         NumFixedGPRs += 1;
         RC = &SystemZ::GR32BitRegClass;
         break;
       case MVT::i64:
         NumFixedGPRs += 1;
         RC = &SystemZ::GR64BitRegClass;
         break;
       case MVT::f32:
         NumFixedFPRs += 1;
         RC = &SystemZ::FP32BitRegClass;
         break;
       case MVT::f64:
         NumFixedFPRs += 1;
         RC = &SystemZ::FP64BitRegClass;
         break;
       case MVT::f128:
         NumFixedFPRs += 2;
         RC = &SystemZ::FP128BitRegClass;
         break;
       case MVT::v16i8:
       case MVT::v8i16:
       case MVT::v4i32:
       case MVT::v2i64:
       case MVT::v4f32:
       case MVT::v2f64:
         RC = &SystemZ::VR128BitRegClass;
         break;
       }
 
       Register VReg = MRI.createVirtualRegister(RC);
       MRI.addLiveIn(VA.getLocReg(), VReg);
       ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
     } else {
       assert(VA.isMemLoc() && "Argument not register or memory");
 
       // Create the frame index object for this incoming parameter.
       // FIXME: Pre-include call frame size in the offset, should not
       // need to manually add it here.
       int64_t ArgSPOffset = VA.getLocMemOffset();
       if (Subtarget.isTargetXPLINK64()) {
         auto &XPRegs =
             Subtarget.getSpecialRegisters<SystemZXPLINK64Registers>();
         ArgSPOffset += XPRegs.getCallFrameSize();
       }
       int FI =
           MFI.CreateFixedObject(LocVT.getSizeInBits() / 8, ArgSPOffset, true);
 
       // Create the SelectionDAG nodes corresponding to a load
       // from this parameter.  Unpromoted ints and floats are
       // passed as right-justified 8-byte values.
       SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
       if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
         FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
                           DAG.getIntPtrConstant(4, DL));
       ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN,
                              MachinePointerInfo::getFixedStack(MF, FI));
     }
 
     // Convert the value of the argument register into the value that's
     // being passed.
     if (VA.getLocInfo() == CCValAssign::Indirect) {
       InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
                                    MachinePointerInfo()));
       // If the original argument was split (e.g. i128), we need
       // to load all parts of it here (using the same address).
       unsigned ArgIndex = Ins[I].OrigArgIndex;
       assert (Ins[I].PartOffset == 0);
       while (I + 1 != E && Ins[I + 1].OrigArgIndex == ArgIndex) {
         CCValAssign &PartVA = ArgLocs[I + 1];
         unsigned PartOffset = Ins[I + 1].PartOffset;
         SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
                                       DAG.getIntPtrConstant(PartOffset, DL));
         InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
                                      MachinePointerInfo()));
         ++I;
       }
     } else
       InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue));
   }
 
   // FIXME: Add support for lowering varargs for XPLINK64 in a later patch.
   if (IsVarArg && Subtarget.isTargetELF()) {
     // Save the number of non-varargs registers for later use by va_start, etc.
     FuncInfo->setVarArgsFirstGPR(NumFixedGPRs);
     FuncInfo->setVarArgsFirstFPR(NumFixedFPRs);
 
     // Likewise the address (in the form of a frame index) of where the
     // first stack vararg would be.  The 1-byte size here is arbitrary.
     int64_t StackSize = CCInfo.getNextStackOffset();
     FuncInfo->setVarArgsFrameIndex(MFI.CreateFixedObject(1, StackSize, true));
 
     // ...and a similar frame index for the caller-allocated save area
     // that will be used to store the incoming registers.
     int64_t RegSaveOffset =
       -SystemZMC::ELFCallFrameSize + TFL->getRegSpillOffset(MF, SystemZ::R2D) - 16;
     unsigned RegSaveIndex = MFI.CreateFixedObject(1, RegSaveOffset, true);
     FuncInfo->setRegSaveFrameIndex(RegSaveIndex);
 
     // Store the FPR varargs in the reserved frame slots.  (We store the
     // GPRs as part of the prologue.)
     if (NumFixedFPRs < SystemZ::ELFNumArgFPRs && !useSoftFloat()) {
       SDValue MemOps[SystemZ::ELFNumArgFPRs];
       for (unsigned I = NumFixedFPRs; I < SystemZ::ELFNumArgFPRs; ++I) {
         unsigned Offset = TFL->getRegSpillOffset(MF, SystemZ::ELFArgFPRs[I]);
         int FI =
           MFI.CreateFixedObject(8, -SystemZMC::ELFCallFrameSize + Offset, true);
         SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
         Register VReg = MF.addLiveIn(SystemZ::ELFArgFPRs[I],
                                      &SystemZ::FP64BitRegClass);
         SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64);
         MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN,
                                  MachinePointerInfo::getFixedStack(MF, FI));
       }
       // Join the stores, which are independent of one another.
       Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
                           makeArrayRef(&MemOps[NumFixedFPRs],
                                        SystemZ::ELFNumArgFPRs-NumFixedFPRs));
     }
   }
 
   // FIXME: For XPLINK64, Add in support for handling incoming "ADA" special
   // register (R5)
   return Chain;
 }
 
 static bool canUseSiblingCall(const CCState &ArgCCInfo,
                               SmallVectorImpl<CCValAssign> &ArgLocs,
                               SmallVectorImpl<ISD::OutputArg> &Outs) {
   // Punt if there are any indirect or stack arguments, or if the call
   // needs the callee-saved argument register R6, or if the call uses
   // the callee-saved register arguments SwiftSelf and SwiftError.
   for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
     CCValAssign &VA = ArgLocs[I];
     if (VA.getLocInfo() == CCValAssign::Indirect)
       return false;
     if (!VA.isRegLoc())
       return false;
     Register Reg = VA.getLocReg();
     if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D)
       return false;
     if (Outs[I].Flags.isSwiftSelf() || Outs[I].Flags.isSwiftError())
       return false;
   }
   return true;
 }
 
 SDValue
 SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
                                  SmallVectorImpl<SDValue> &InVals) const {
   SelectionDAG &DAG = CLI.DAG;
   SDLoc &DL = CLI.DL;
   SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
   SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
   SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
   SDValue Chain = CLI.Chain;
   SDValue Callee = CLI.Callee;
   bool &IsTailCall = CLI.IsTailCall;
   CallingConv::ID CallConv = CLI.CallConv;
   bool IsVarArg = CLI.IsVarArg;
   MachineFunction &MF = DAG.getMachineFunction();
   EVT PtrVT = getPointerTy(MF.getDataLayout());
   LLVMContext &Ctx = *DAG.getContext();
   SystemZCallingConventionRegisters *Regs = Subtarget.getSpecialRegisters();
 
   // FIXME: z/OS support to be added in later.
   if (Subtarget.isTargetXPLINK64())
     IsTailCall = false;
 
   // Detect unsupported vector argument and return types.
   if (Subtarget.hasVector()) {
     VerifyVectorTypes(Outs);
     VerifyVectorTypes(Ins);
   }
 
   // Analyze the operands of the call, assigning locations to each operand.
   SmallVector<CCValAssign, 16> ArgLocs;
   SystemZCCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, Ctx);
   ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ);
 
   // We don't support GuaranteedTailCallOpt, only automatically-detected
   // sibling calls.
   if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs, Outs))
     IsTailCall = false;
 
   // Get a count of how many bytes are to be pushed on the stack.
   unsigned NumBytes = ArgCCInfo.getNextStackOffset();
 
   if (Subtarget.isTargetXPLINK64())
     // Although the XPLINK specifications for AMODE64 state that minimum size
     // of the param area is minimum 32 bytes and no rounding is otherwise
     // specified, we round this area in 64 bytes increments to be compatible
     // with existing compilers.
     NumBytes = std::max(64U, (unsigned)alignTo(NumBytes, 64));
 
   // Mark the start of the call.
   if (!IsTailCall)
     Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL);
 
   // Copy argument values to their designated locations.
   SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass;
   SmallVector<SDValue, 8> MemOpChains;
   SDValue StackPtr;
   for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
     CCValAssign &VA = ArgLocs[I];
     SDValue ArgValue = OutVals[I];
 
     if (VA.getLocInfo() == CCValAssign::Indirect) {
       // Store the argument in a stack slot and pass its address.
       unsigned ArgIndex = Outs[I].OrigArgIndex;
       EVT SlotVT;
       if (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) {
         // Allocate the full stack space for a promoted (and split) argument.
         Type *OrigArgType = CLI.Args[Outs[I].OrigArgIndex].Ty;
         EVT OrigArgVT = getValueType(MF.getDataLayout(), OrigArgType);
         MVT PartVT = getRegisterTypeForCallingConv(Ctx, CLI.CallConv, OrigArgVT);
         unsigned N = getNumRegistersForCallingConv(Ctx, CLI.CallConv, OrigArgVT);
         SlotVT = EVT::getIntegerVT(Ctx, PartVT.getSizeInBits() * N);
       } else {
         SlotVT = Outs[I].ArgVT;
       }
       SDValue SpillSlot = DAG.CreateStackTemporary(SlotVT);
       int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
       MemOpChains.push_back(
           DAG.getStore(Chain, DL, ArgValue, SpillSlot,
                        MachinePointerInfo::getFixedStack(MF, FI)));
       // If the original argument was split (e.g. i128), we need
       // to store all parts of it here (and pass just one address).
       assert (Outs[I].PartOffset == 0);
       while (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) {
         SDValue PartValue = OutVals[I + 1];
         unsigned PartOffset = Outs[I + 1].PartOffset;
         SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
                                       DAG.getIntPtrConstant(PartOffset, DL));
         MemOpChains.push_back(
             DAG.getStore(Chain, DL, PartValue, Address,
                          MachinePointerInfo::getFixedStack(MF, FI)));
         assert((PartOffset + PartValue.getValueType().getStoreSize() <=
                 SlotVT.getStoreSize()) && "Not enough space for argument part!");
         ++I;
       }
       ArgValue = SpillSlot;
     } else
       ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue);
 
     if (VA.isRegLoc()) {
       // In XPLINK64, for the 128-bit vararg case, ArgValue is bitcasted to a
       // MVT::i128 type. We decompose the 128-bit type to a pair of its high
       // and low values.
       if (VA.getLocVT() == MVT::i128)
         ArgValue = lowerI128ToGR128(DAG, ArgValue);
       // Queue up the argument copies and emit them at the end.
       RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
     } else {
       assert(VA.isMemLoc() && "Argument not register or memory");
 
       // Work out the address of the stack slot.  Unpromoted ints and
       // floats are passed as right-justified 8-byte values.
       if (!StackPtr.getNode())
         StackPtr = DAG.getCopyFromReg(Chain, DL,
                                       Regs->getStackPointerRegister(), PtrVT);
       unsigned Offset = Regs->getStackPointerBias() + Regs->getCallFrameSize() +
                         VA.getLocMemOffset();
       if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32)
         Offset += 4;
       SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
                                     DAG.getIntPtrConstant(Offset, DL));
 
       // Emit the store.
       MemOpChains.push_back(
           DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
 
       // Although long doubles or vectors are passed through the stack when
       // they are vararg (non-fixed arguments), if a long double or vector
       // occupies the third and fourth slot of the argument list GPR3 should
       // still shadow the third slot of the argument list.
       if (Subtarget.isTargetXPLINK64() && VA.needsCustom()) {
         SDValue ShadowArgValue =
             DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, ArgValue,
                         DAG.getIntPtrConstant(1, DL));
         RegsToPass.push_back(std::make_pair(SystemZ::R3D, ShadowArgValue));
       }
     }
   }
 
   // Join the stores, which are independent of one another.
   if (!MemOpChains.empty())
     Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
 
   // Accept direct calls by converting symbolic call addresses to the
   // associated Target* opcodes.  Force %r1 to be used for indirect
   // tail calls.
   SDValue Glue;
   // FIXME: Add support for XPLINK using the ADA register.
   if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
     Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT);
     Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
   } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
     Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT);
     Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee);
   } else if (IsTailCall) {
     Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue);
     Glue = Chain.getValue(1);
     Callee = DAG.getRegister(SystemZ::R1D, Callee.getValueType());
   }
 
   // Build a sequence of copy-to-reg nodes, chained and glued together.
   for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) {
     Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first,
                              RegsToPass[I].second, Glue);
     Glue = Chain.getValue(1);
   }
 
   // The first call operand is the chain and the second is the target address.
   SmallVector<SDValue, 8> Ops;
   Ops.push_back(Chain);
   Ops.push_back(Callee);
 
   // Add argument registers to the end of the list so that they are
   // known live into the call.
   for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I)
     Ops.push_back(DAG.getRegister(RegsToPass[I].first,
                                   RegsToPass[I].second.getValueType()));
 
   // Add a register mask operand representing the call-preserved registers.
   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
   const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
   assert(Mask && "Missing call preserved mask for calling convention");
   Ops.push_back(DAG.getRegisterMask(Mask));
 
   // Glue the call to the argument copies, if any.
   if (Glue.getNode())
     Ops.push_back(Glue);
 
   // Emit the call.
   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
   if (IsTailCall)
     return DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, Ops);
   Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, Ops);
   DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
   Glue = Chain.getValue(1);
 
   // Mark the end of the call, which is glued to the call itself.
   Chain = DAG.getCALLSEQ_END(Chain,
                              DAG.getConstant(NumBytes, DL, PtrVT, true),
                              DAG.getConstant(0, DL, PtrVT, true),
                              Glue, DL);
   Glue = Chain.getValue(1);
 
   // Assign locations to each value returned by this call.
   SmallVector<CCValAssign, 16> RetLocs;
   CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Ctx);
   RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ);
 
   // Copy all of the result registers out of their specified physreg.
   for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) {
     CCValAssign &VA = RetLocs[I];
 
     // Copy the value out, gluing the copy to the end of the call sequence.
     SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(),
                                           VA.getLocVT(), Glue);
     Chain = RetValue.getValue(1);
     Glue = RetValue.getValue(2);
 
     // Convert the value of the return register into the value that's
     // being returned.
     InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue));
   }
 
   return Chain;
 }
 
 bool SystemZTargetLowering::
 CanLowerReturn(CallingConv::ID CallConv,
                MachineFunction &MF, bool isVarArg,
                const SmallVectorImpl<ISD::OutputArg> &Outs,
                LLVMContext &Context) const {
   // Detect unsupported vector return types.
   if (Subtarget.hasVector())
     VerifyVectorTypes(Outs);
 
   // Special case that we cannot easily detect in RetCC_SystemZ since
   // i128 is not a legal type.
   for (auto &Out : Outs)
     if (Out.ArgVT == MVT::i128)
       return false;
 
   SmallVector<CCValAssign, 16> RetLocs;
   CCState RetCCInfo(CallConv, isVarArg, MF, RetLocs, Context);
   return RetCCInfo.CheckReturn(Outs, RetCC_SystemZ);
 }
 
 SDValue
 SystemZTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
                                    bool IsVarArg,
                                    const SmallVectorImpl<ISD::OutputArg> &Outs,
                                    const SmallVectorImpl<SDValue> &OutVals,
                                    const SDLoc &DL, SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
 
   // Detect unsupported vector return types.
   if (Subtarget.hasVector())
     VerifyVectorTypes(Outs);
 
   // Assign locations to each returned value.
   SmallVector<CCValAssign, 16> RetLocs;
   CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext());
   RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ);
 
   // Quick exit for void returns
   if (RetLocs.empty())
     return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, Chain);
 
   if (CallConv == CallingConv::GHC)
     report_fatal_error("GHC functions return void only");
 
   // Copy the result values into the output registers.
   SDValue Glue;
   SmallVector<SDValue, 4> RetOps;
   RetOps.push_back(Chain);
   for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) {
     CCValAssign &VA = RetLocs[I];
     SDValue RetValue = OutVals[I];
 
     // Make the return register live on exit.
     assert(VA.isRegLoc() && "Can only return in registers!");
 
     // Promote the value as required.
     RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue);
 
     // Chain and glue the copies together.
     Register Reg = VA.getLocReg();
     Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue);
     Glue = Chain.getValue(1);
     RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT()));
   }
 
   // Update chain and glue.
   RetOps[0] = Chain;
   if (Glue.getNode())
     RetOps.push_back(Glue);
 
   return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, RetOps);
 }
 
 // Return true if Op is an intrinsic node with chain that returns the CC value
 // as its only (other) argument.  Provide the associated SystemZISD opcode and
 // the mask of valid CC values if so.
 static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode,
                                       unsigned &CCValid) {
   unsigned Id = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
   switch (Id) {
   case Intrinsic::s390_tbegin:
     Opcode = SystemZISD::TBEGIN;
     CCValid = SystemZ::CCMASK_TBEGIN;
     return true;
 
   case Intrinsic::s390_tbegin_nofloat:
     Opcode = SystemZISD::TBEGIN_NOFLOAT;
     CCValid = SystemZ::CCMASK_TBEGIN;
     return true;
 
   case Intrinsic::s390_tend:
     Opcode = SystemZISD::TEND;
     CCValid = SystemZ::CCMASK_TEND;
     return true;
 
   default:
     return false;
   }
 }
 
 // Return true if Op is an intrinsic node without chain that returns the
 // CC value as its final argument.  Provide the associated SystemZISD
 // opcode and the mask of valid CC values if so.
 static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid) {
   unsigned Id = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   switch (Id) {
   case Intrinsic::s390_vpkshs:
   case Intrinsic::s390_vpksfs:
   case Intrinsic::s390_vpksgs:
     Opcode = SystemZISD::PACKS_CC;
     CCValid = SystemZ::CCMASK_VCMP;
     return true;
 
   case Intrinsic::s390_vpklshs:
   case Intrinsic::s390_vpklsfs:
   case Intrinsic::s390_vpklsgs:
     Opcode = SystemZISD::PACKLS_CC;
     CCValid = SystemZ::CCMASK_VCMP;
     return true;
 
   case Intrinsic::s390_vceqbs:
   case Intrinsic::s390_vceqhs:
   case Intrinsic::s390_vceqfs:
   case Intrinsic::s390_vceqgs:
     Opcode = SystemZISD::VICMPES;
     CCValid = SystemZ::CCMASK_VCMP;
     return true;
 
   case Intrinsic::s390_vchbs:
   case Intrinsic::s390_vchhs:
   case Intrinsic::s390_vchfs:
   case Intrinsic::s390_vchgs:
     Opcode = SystemZISD::VICMPHS;
     CCValid = SystemZ::CCMASK_VCMP;
     return true;
 
   case Intrinsic::s390_vchlbs:
   case Intrinsic::s390_vchlhs:
   case Intrinsic::s390_vchlfs:
   case Intrinsic::s390_vchlgs:
     Opcode = SystemZISD::VICMPHLS;
     CCValid = SystemZ::CCMASK_VCMP;
     return true;
 
   case Intrinsic::s390_vtm:
     Opcode = SystemZISD::VTM;
     CCValid = SystemZ::CCMASK_VCMP;
     return true;
 
   case Intrinsic::s390_vfaebs:
   case Intrinsic::s390_vfaehs:
   case Intrinsic::s390_vfaefs:
     Opcode = SystemZISD::VFAE_CC;
     CCValid = SystemZ::CCMASK_ANY;
     return true;
 
   case Intrinsic::s390_vfaezbs:
   case Intrinsic::s390_vfaezhs:
   case Intrinsic::s390_vfaezfs:
     Opcode = SystemZISD::VFAEZ_CC;
     CCValid = SystemZ::CCMASK_ANY;
     return true;
 
   case Intrinsic::s390_vfeebs:
   case Intrinsic::s390_vfeehs:
   case Intrinsic::s390_vfeefs:
     Opcode = SystemZISD::VFEE_CC;
     CCValid = SystemZ::CCMASK_ANY;
     return true;
 
   case Intrinsic::s390_vfeezbs:
   case Intrinsic::s390_vfeezhs:
   case Intrinsic::s390_vfeezfs:
     Opcode = SystemZISD::VFEEZ_CC;
     CCValid = SystemZ::CCMASK_ANY;
     return true;
 
   case Intrinsic::s390_vfenebs:
   case Intrinsic::s390_vfenehs:
   case Intrinsic::s390_vfenefs:
     Opcode = SystemZISD::VFENE_CC;
     CCValid = SystemZ::CCMASK_ANY;
     return true;
 
   case Intrinsic::s390_vfenezbs:
   case Intrinsic::s390_vfenezhs:
   case Intrinsic::s390_vfenezfs:
     Opcode = SystemZISD::VFENEZ_CC;
     CCValid = SystemZ::CCMASK_ANY;
     return true;
 
   case Intrinsic::s390_vistrbs:
   case Intrinsic::s390_vistrhs:
   case Intrinsic::s390_vistrfs:
     Opcode = SystemZISD::VISTR_CC;
     CCValid = SystemZ::CCMASK_0 | SystemZ::CCMASK_3;
     return true;
 
   case Intrinsic::s390_vstrcbs:
   case Intrinsic::s390_vstrchs:
   case Intrinsic::s390_vstrcfs:
     Opcode = SystemZISD::VSTRC_CC;
     CCValid = SystemZ::CCMASK_ANY;
     return true;
 
   case Intrinsic::s390_vstrczbs:
   case Intrinsic::s390_vstrczhs:
   case Intrinsic::s390_vstrczfs:
     Opcode = SystemZISD::VSTRCZ_CC;
     CCValid = SystemZ::CCMASK_ANY;
     return true;
 
   case Intrinsic::s390_vstrsb:
   case Intrinsic::s390_vstrsh:
   case Intrinsic::s390_vstrsf:
     Opcode = SystemZISD::VSTRS_CC;
     CCValid = SystemZ::CCMASK_ANY;
     return true;
 
   case Intrinsic::s390_vstrszb:
   case Intrinsic::s390_vstrszh:
   case Intrinsic::s390_vstrszf:
     Opcode = SystemZISD::VSTRSZ_CC;
     CCValid = SystemZ::CCMASK_ANY;
     return true;
 
   case Intrinsic::s390_vfcedbs:
   case Intrinsic::s390_vfcesbs:
     Opcode = SystemZISD::VFCMPES;
     CCValid = SystemZ::CCMASK_VCMP;
     return true;
 
   case Intrinsic::s390_vfchdbs:
   case Intrinsic::s390_vfchsbs:
     Opcode = SystemZISD::VFCMPHS;
     CCValid = SystemZ::CCMASK_VCMP;
     return true;
 
   case Intrinsic::s390_vfchedbs:
   case Intrinsic::s390_vfchesbs:
     Opcode = SystemZISD::VFCMPHES;
     CCValid = SystemZ::CCMASK_VCMP;
     return true;
 
   case Intrinsic::s390_vftcidb:
   case Intrinsic::s390_vftcisb:
     Opcode = SystemZISD::VFTCI;
     CCValid = SystemZ::CCMASK_VCMP;
     return true;
 
   case Intrinsic::s390_tdc:
     Opcode = SystemZISD::TDC;
     CCValid = SystemZ::CCMASK_TDC;
     return true;
 
   default:
     return false;
   }
 }
 
 // Emit an intrinsic with chain and an explicit CC register result.
 static SDNode *emitIntrinsicWithCCAndChain(SelectionDAG &DAG, SDValue Op,
                                            unsigned Opcode) {
   // Copy all operands except the intrinsic ID.
   unsigned NumOps = Op.getNumOperands();
   SmallVector<SDValue, 6> Ops;
   Ops.reserve(NumOps - 1);
   Ops.push_back(Op.getOperand(0));
   for (unsigned I = 2; I < NumOps; ++I)
     Ops.push_back(Op.getOperand(I));
 
   assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
   SDVTList RawVTs = DAG.getVTList(MVT::i32, MVT::Other);
   SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), RawVTs, Ops);
   SDValue OldChain = SDValue(Op.getNode(), 1);
   SDValue NewChain = SDValue(Intr.getNode(), 1);
   DAG.ReplaceAllUsesOfValueWith(OldChain, NewChain);
   return Intr.getNode();
 }
 
 // Emit an intrinsic with an explicit CC register result.
 static SDNode *emitIntrinsicWithCC(SelectionDAG &DAG, SDValue Op,
                                    unsigned Opcode) {
   // Copy all operands except the intrinsic ID.
   unsigned NumOps = Op.getNumOperands();
   SmallVector<SDValue, 6> Ops;
   Ops.reserve(NumOps - 1);
   for (unsigned I = 1; I < NumOps; ++I)
     Ops.push_back(Op.getOperand(I));
 
   SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), Op->getVTList(), Ops);
   return Intr.getNode();
 }
 
 // CC is a comparison that will be implemented using an integer or
 // floating-point comparison.  Return the condition code mask for
 // a branch on true.  In the integer case, CCMASK_CMP_UO is set for
 // unsigned comparisons and clear for signed ones.  In the floating-point
 // case, CCMASK_CMP_UO has its normal mask meaning (unordered).
 static unsigned CCMaskForCondCode(ISD::CondCode CC) {
 #define CONV(X) \
   case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
   case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
   case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
 
   switch (CC) {
   default:
     llvm_unreachable("Invalid integer condition!");
 
   CONV(EQ);
   CONV(NE);
   CONV(GT);
   CONV(GE);
   CONV(LT);
   CONV(LE);
 
   case ISD::SETO:  return SystemZ::CCMASK_CMP_O;
   case ISD::SETUO: return SystemZ::CCMASK_CMP_UO;
   }
 #undef CONV
 }
 
 // If C can be converted to a comparison against zero, adjust the operands
 // as necessary.
 static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
   if (C.ICmpType == SystemZICMP::UnsignedOnly)
     return;
 
   auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1.getNode());
   if (!ConstOp1)
     return;
 
   int64_t Value = ConstOp1->getSExtValue();
   if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) ||
       (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) ||
       (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) ||
       (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) {
     C.CCMask ^= SystemZ::CCMASK_CMP_EQ;
     C.Op1 = DAG.getConstant(0, DL, C.Op1.getValueType());
   }
 }
 
 // If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI,
 // adjust the operands as necessary.
 static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL,
                              Comparison &C) {
   // For us to make any changes, it must a comparison between a single-use
   // load and a constant.
   if (!C.Op0.hasOneUse() ||
       C.Op0.getOpcode() != ISD::LOAD ||
       C.Op1.getOpcode() != ISD::Constant)
     return;
 
   // We must have an 8- or 16-bit load.
   auto *Load = cast<LoadSDNode>(C.Op0);
   unsigned NumBits = Load->getMemoryVT().getSizeInBits();
   if ((NumBits != 8 && NumBits != 16) ||
       NumBits != Load->getMemoryVT().getStoreSizeInBits())
     return;
 
   // The load must be an extending one and the constant must be within the
   // range of the unextended value.
   auto *ConstOp1 = cast<ConstantSDNode>(C.Op1);
   uint64_t Value = ConstOp1->getZExtValue();
   uint64_t Mask = (1 << NumBits) - 1;
   if (Load->getExtensionType() == ISD::SEXTLOAD) {
     // Make sure that ConstOp1 is in range of C.Op0.
     int64_t SignedValue = ConstOp1->getSExtValue();
     if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask)
       return;
     if (C.ICmpType != SystemZICMP::SignedOnly) {
       // Unsigned comparison between two sign-extended values is equivalent
       // to unsigned comparison between two zero-extended values.
       Value &= Mask;
     } else if (NumBits == 8) {
       // Try to treat the comparison as unsigned, so that we can use CLI.
       // Adjust CCMask and Value as necessary.
       if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT)
         // Test whether the high bit of the byte is set.
         Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT;
       else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE)
         // Test whether the high bit of the byte is clear.
         Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT;
       else
         // No instruction exists for this combination.
         return;
       C.ICmpType = SystemZICMP::UnsignedOnly;
     }
   } else if (Load->getExtensionType() == ISD::ZEXTLOAD) {
     if (Value > Mask)
       return;
     // If the constant is in range, we can use any comparison.
     C.ICmpType = SystemZICMP::Any;
   } else
     return;
 
   // Make sure that the first operand is an i32 of the right extension type.
   ISD::LoadExtType ExtType = (C.ICmpType == SystemZICMP::SignedOnly ?
                               ISD::SEXTLOAD :
                               ISD::ZEXTLOAD);
   if (C.Op0.getValueType() != MVT::i32 ||
       Load->getExtensionType() != ExtType) {
     C.Op0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, Load->getChain(),
                            Load->getBasePtr(), Load->getPointerInfo(),
                            Load->getMemoryVT(), Load->getAlignment(),
                            Load->getMemOperand()->getFlags());
     // Update the chain uses.
     DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), C.Op0.getValue(1));
   }
 
   // Make sure that the second operand is an i32 with the right value.
   if (C.Op1.getValueType() != MVT::i32 ||
       Value != ConstOp1->getZExtValue())
     C.Op1 = DAG.getConstant(Value, DL, MVT::i32);
 }
 
 // Return true if Op is either an unextended load, or a load suitable
 // for integer register-memory comparisons of type ICmpType.
 static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) {
   auto *Load = dyn_cast<LoadSDNode>(Op.getNode());
   if (Load) {
     // There are no instructions to compare a register with a memory byte.
     if (Load->getMemoryVT() == MVT::i8)
       return false;
     // Otherwise decide on extension type.
     switch (Load->getExtensionType()) {
     case ISD::NON_EXTLOAD:
       return true;
     case ISD::SEXTLOAD:
       return ICmpType != SystemZICMP::UnsignedOnly;
     case ISD::ZEXTLOAD:
       return ICmpType != SystemZICMP::SignedOnly;
     default:
       break;
     }
   }
   return false;
 }
 
 // Return true if it is better to swap the operands of C.
 static bool shouldSwapCmpOperands(const Comparison &C) {
   // Leave f128 comparisons alone, since they have no memory forms.
   if (C.Op0.getValueType() == MVT::f128)
     return false;
 
   // Always keep a floating-point constant second, since comparisons with
   // zero can use LOAD TEST and comparisons with other constants make a
   // natural memory operand.
   if (isa<ConstantFPSDNode>(C.Op1))
     return false;
 
   // Never swap comparisons with zero since there are many ways to optimize
   // those later.
   auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1);
   if (ConstOp1 && ConstOp1->getZExtValue() == 0)
     return false;
 
   // Also keep natural memory operands second if the loaded value is
   // only used here.  Several comparisons have memory forms.
   if (isNaturalMemoryOperand(C.Op1, C.ICmpType) && C.Op1.hasOneUse())
     return false;
 
   // Look for cases where Cmp0 is a single-use load and Cmp1 isn't.
   // In that case we generally prefer the memory to be second.
   if (isNaturalMemoryOperand(C.Op0, C.ICmpType) && C.Op0.hasOneUse()) {
     // The only exceptions are when the second operand is a constant and
     // we can use things like CHHSI.
     if (!ConstOp1)
       return true;
     // The unsigned memory-immediate instructions can handle 16-bit
     // unsigned integers.
     if (C.ICmpType != SystemZICMP::SignedOnly &&
         isUInt<16>(ConstOp1->getZExtValue()))
       return false;
     // The signed memory-immediate instructions can handle 16-bit
     // signed integers.
     if (C.ICmpType != SystemZICMP::UnsignedOnly &&
         isInt<16>(ConstOp1->getSExtValue()))
       return false;
     return true;
   }
 
   // Try to promote the use of CGFR and CLGFR.
   unsigned Opcode0 = C.Op0.getOpcode();
   if (C.ICmpType != SystemZICMP::UnsignedOnly && Opcode0 == ISD::SIGN_EXTEND)
     return true;
   if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::ZERO_EXTEND)
     return true;
   if (C.ICmpType != SystemZICMP::SignedOnly &&
       Opcode0 == ISD::AND &&
       C.Op0.getOperand(1).getOpcode() == ISD::Constant &&
       cast<ConstantSDNode>(C.Op0.getOperand(1))->getZExtValue() == 0xffffffff)
     return true;
 
   return false;
 }
 
 // Check whether C tests for equality between X and Y and whether X - Y
 // or Y - X is also computed.  In that case it's better to compare the
 // result of the subtraction against zero.
 static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL,
                                  Comparison &C) {
   if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
       C.CCMask == SystemZ::CCMASK_CMP_NE) {
     for (SDNode *N : C.Op0->uses()) {
       if (N->getOpcode() == ISD::SUB &&
           ((N->getOperand(0) == C.Op0 && N->getOperand(1) == C.Op1) ||
            (N->getOperand(0) == C.Op1 && N->getOperand(1) == C.Op0))) {
         C.Op0 = SDValue(N, 0);
         C.Op1 = DAG.getConstant(0, DL, N->getValueType(0));
         return;
       }
     }
   }
 }
 
 // Check whether C compares a floating-point value with zero and if that
 // floating-point value is also negated.  In this case we can use the
 // negation to set CC, so avoiding separate LOAD AND TEST and
 // LOAD (NEGATIVE/COMPLEMENT) instructions.
 static void adjustForFNeg(Comparison &C) {
   // This optimization is invalid for strict comparisons, since FNEG
   // does not raise any exceptions.
   if (C.Chain)
     return;
   auto *C1 = dyn_cast<ConstantFPSDNode>(C.Op1);
   if (C1 && C1->isZero()) {
     for (SDNode *N : C.Op0->uses()) {
       if (N->getOpcode() == ISD::FNEG) {
         C.Op0 = SDValue(N, 0);
         C.CCMask = SystemZ::reverseCCMask(C.CCMask);
         return;
       }
     }
   }
 }
 
 // Check whether C compares (shl X, 32) with 0 and whether X is
 // also sign-extended.  In that case it is better to test the result
 // of the sign extension using LTGFR.
 //
 // This case is important because InstCombine transforms a comparison
 // with (sext (trunc X)) into a comparison with (shl X, 32).
 static void adjustForLTGFR(Comparison &C) {
   // Check for a comparison between (shl X, 32) and 0.
   if (C.Op0.getOpcode() == ISD::SHL &&
       C.Op0.getValueType() == MVT::i64 &&
       C.Op1.getOpcode() == ISD::Constant &&
       cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) {
     auto *C1 = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1));
     if (C1 && C1->getZExtValue() == 32) {
       SDValue ShlOp0 = C.Op0.getOperand(0);
       // See whether X has any SIGN_EXTEND_INREG uses.
       for (SDNode *N : ShlOp0->uses()) {
         if (N->getOpcode() == ISD::SIGN_EXTEND_INREG &&
             cast<VTSDNode>(N->getOperand(1))->getVT() == MVT::i32) {
           C.Op0 = SDValue(N, 0);
           return;
         }
       }
     }
   }
 }
 
 // If C compares the truncation of an extending load, try to compare
 // the untruncated value instead.  This exposes more opportunities to
 // reuse CC.
 static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL,
                                Comparison &C) {
   if (C.Op0.getOpcode() == ISD::TRUNCATE &&
       C.Op0.getOperand(0).getOpcode() == ISD::LOAD &&
       C.Op1.getOpcode() == ISD::Constant &&
       cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) {
     auto *L = cast<LoadSDNode>(C.Op0.getOperand(0));
     if (L->getMemoryVT().getStoreSizeInBits().getFixedSize() <=
         C.Op0.getValueSizeInBits().getFixedSize()) {
       unsigned Type = L->getExtensionType();
       if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) ||
           (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) {
         C.Op0 = C.Op0.getOperand(0);
         C.Op1 = DAG.getConstant(0, DL, C.Op0.getValueType());
       }
     }
   }
 }
 
 // Return true if shift operation N has an in-range constant shift value.
 // Store it in ShiftVal if so.
 static bool isSimpleShift(SDValue N, unsigned &ShiftVal) {
   auto *Shift = dyn_cast<ConstantSDNode>(N.getOperand(1));
   if (!Shift)
     return false;
 
   uint64_t Amount = Shift->getZExtValue();
   if (Amount >= N.getValueSizeInBits())
     return false;
 
   ShiftVal = Amount;
   return true;
 }
 
 // Check whether an AND with Mask is suitable for a TEST UNDER MASK
 // instruction and whether the CC value is descriptive enough to handle
 // a comparison of type Opcode between the AND result and CmpVal.
 // CCMask says which comparison result is being tested and BitSize is
 // the number of bits in the operands.  If TEST UNDER MASK can be used,
 // return the corresponding CC mask, otherwise return 0.
 static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask,
                                      uint64_t Mask, uint64_t CmpVal,
                                      unsigned ICmpType) {
   assert(Mask != 0 && "ANDs with zero should have been removed by now");
 
   // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL.
   if (!SystemZ::isImmLL(Mask) && !SystemZ::isImmLH(Mask) &&
       !SystemZ::isImmHL(Mask) && !SystemZ::isImmHH(Mask))
     return 0;
 
   // Work out the masks for the lowest and highest bits.
   unsigned HighShift = 63 - countLeadingZeros(Mask);
   uint64_t High = uint64_t(1) << HighShift;
   uint64_t Low = uint64_t(1) << countTrailingZeros(Mask);
 
   // Signed ordered comparisons are effectively unsigned if the sign
   // bit is dropped.
   bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly);
 
   // Check for equality comparisons with 0, or the equivalent.
   if (CmpVal == 0) {
     if (CCMask == SystemZ::CCMASK_CMP_EQ)
       return SystemZ::CCMASK_TM_ALL_0;
     if (CCMask == SystemZ::CCMASK_CMP_NE)
       return SystemZ::CCMASK_TM_SOME_1;
   }
   if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <= Low) {
     if (CCMask == SystemZ::CCMASK_CMP_LT)
       return SystemZ::CCMASK_TM_ALL_0;
     if (CCMask == SystemZ::CCMASK_CMP_GE)
       return SystemZ::CCMASK_TM_SOME_1;
   }
   if (EffectivelyUnsigned && CmpVal < Low) {
     if (CCMask == SystemZ::CCMASK_CMP_LE)
       return SystemZ::CCMASK_TM_ALL_0;
     if (CCMask == SystemZ::CCMASK_CMP_GT)
       return SystemZ::CCMASK_TM_SOME_1;
   }
 
   // Check for equality comparisons with the mask, or the equivalent.
   if (CmpVal == Mask) {
     if (CCMask == SystemZ::CCMASK_CMP_EQ)
       return SystemZ::CCMASK_TM_ALL_1;
     if (CCMask == SystemZ::CCMASK_CMP_NE)
       return SystemZ::CCMASK_TM_SOME_0;
   }
   if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) {
     if (CCMask == SystemZ::CCMASK_CMP_GT)
       return SystemZ::CCMASK_TM_ALL_1;
     if (CCMask == SystemZ::CCMASK_CMP_LE)
       return SystemZ::CCMASK_TM_SOME_0;
   }
   if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) {
     if (CCMask == SystemZ::CCMASK_CMP_GE)
       return SystemZ::CCMASK_TM_ALL_1;
     if (CCMask == SystemZ::CCMASK_CMP_LT)
       return SystemZ::CCMASK_TM_SOME_0;
   }
 
   // Check for ordered comparisons with the top bit.
   if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) {
     if (CCMask == SystemZ::CCMASK_CMP_LE)
       return SystemZ::CCMASK_TM_MSB_0;
     if (CCMask == SystemZ::CCMASK_CMP_GT)
       return SystemZ::CCMASK_TM_MSB_1;
   }
   if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) {
     if (CCMask == SystemZ::CCMASK_CMP_LT)
       return SystemZ::CCMASK_TM_MSB_0;
     if (CCMask == SystemZ::CCMASK_CMP_GE)
       return SystemZ::CCMASK_TM_MSB_1;
   }
 
   // If there are just two bits, we can do equality checks for Low and High
   // as well.
   if (Mask == Low + High) {
     if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low)
       return SystemZ::CCMASK_TM_MIXED_MSB_0;
     if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low)
       return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY;
     if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High)
       return SystemZ::CCMASK_TM_MIXED_MSB_1;
     if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High)
       return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY;
   }
 
   // Looks like we've exhausted our options.
   return 0;
 }
 
 // See whether C can be implemented as a TEST UNDER MASK instruction.
 // Update the arguments with the TM version if so.
 static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL,
                                    Comparison &C) {
   // Check that we have a comparison with a constant.
   auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1);
   if (!ConstOp1)
     return;
   uint64_t CmpVal = ConstOp1->getZExtValue();
 
   // Check whether the nonconstant input is an AND with a constant mask.
   Comparison NewC(C);
   uint64_t MaskVal;
   ConstantSDNode *Mask = nullptr;
   if (C.Op0.getOpcode() == ISD::AND) {
     NewC.Op0 = C.Op0.getOperand(0);
     NewC.Op1 = C.Op0.getOperand(1);
     Mask = dyn_cast<ConstantSDNode>(NewC.Op1);
     if (!Mask)
       return;
     MaskVal = Mask->getZExtValue();
   } else {
     // There is no instruction to compare with a 64-bit immediate
     // so use TMHH instead if possible.  We need an unsigned ordered
     // comparison with an i64 immediate.
     if (NewC.Op0.getValueType() != MVT::i64 ||
         NewC.CCMask == SystemZ::CCMASK_CMP_EQ ||
         NewC.CCMask == SystemZ::CCMASK_CMP_NE ||
         NewC.ICmpType == SystemZICMP::SignedOnly)
       return;
     // Convert LE and GT comparisons into LT and GE.
     if (NewC.CCMask == SystemZ::CCMASK_CMP_LE ||
         NewC.CCMask == SystemZ::CCMASK_CMP_GT) {
       if (CmpVal == uint64_t(-1))
         return;
       CmpVal += 1;
       NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ;
     }
     // If the low N bits of Op1 are zero than the low N bits of Op0 can
     // be masked off without changing the result.
     MaskVal = -(CmpVal & -CmpVal);
     NewC.ICmpType = SystemZICMP::UnsignedOnly;
   }
   if (!MaskVal)
     return;
 
   // Check whether the combination of mask, comparison value and comparison
   // type are suitable.
   unsigned BitSize = NewC.Op0.getValueSizeInBits();
   unsigned NewCCMask, ShiftVal;
   if (NewC.ICmpType != SystemZICMP::SignedOnly &&
       NewC.Op0.getOpcode() == ISD::SHL &&
       isSimpleShift(NewC.Op0, ShiftVal) &&
       (MaskVal >> ShiftVal != 0) &&
       ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal &&
       (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask,
                                         MaskVal >> ShiftVal,
                                         CmpVal >> ShiftVal,
                                         SystemZICMP::Any))) {
     NewC.Op0 = NewC.Op0.getOperand(0);
     MaskVal >>= ShiftVal;
   } else if (NewC.ICmpType != SystemZICMP::SignedOnly &&
              NewC.Op0.getOpcode() == ISD::SRL &&
              isSimpleShift(NewC.Op0, ShiftVal) &&
              (MaskVal << ShiftVal != 0) &&
              ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal &&
              (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask,
                                                MaskVal << ShiftVal,
                                                CmpVal << ShiftVal,
                                                SystemZICMP::UnsignedOnly))) {
     NewC.Op0 = NewC.Op0.getOperand(0);
     MaskVal <<= ShiftVal;
   } else {
     NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, MaskVal, CmpVal,
                                      NewC.ICmpType);
     if (!NewCCMask)
       return;
   }
 
   // Go ahead and make the change.
   C.Opcode = SystemZISD::TM;
   C.Op0 = NewC.Op0;
   if (Mask && Mask->getZExtValue() == MaskVal)
     C.Op1 = SDValue(Mask, 0);
   else
     C.Op1 = DAG.getConstant(MaskVal, DL, C.Op0.getValueType());
   C.CCValid = SystemZ::CCMASK_TM;
   C.CCMask = NewCCMask;
 }
 
 // See whether the comparison argument contains a redundant AND
 // and remove it if so.  This sometimes happens due to the generic
 // BRCOND expansion.
 static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL,
                                   Comparison &C) {
   if (C.Op0.getOpcode() != ISD::AND)
     return;
   auto *Mask = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1));
   if (!Mask)
     return;
   KnownBits Known = DAG.computeKnownBits(C.Op0.getOperand(0));
   if ((~Known.Zero).getZExtValue() & ~Mask->getZExtValue())
     return;
 
   C.Op0 = C.Op0.getOperand(0);
 }
 
 // Return a Comparison that tests the condition-code result of intrinsic
 // node Call against constant integer CC using comparison code Cond.
 // Opcode is the opcode of the SystemZISD operation for the intrinsic
 // and CCValid is the set of possible condition-code results.
 static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode,
                                   SDValue Call, unsigned CCValid, uint64_t CC,
                                   ISD::CondCode Cond) {
   Comparison C(Call, SDValue(), SDValue());
   C.Opcode = Opcode;
   C.CCValid = CCValid;
   if (Cond == ISD::SETEQ)
     // bit 3 for CC==0, bit 0 for CC==3, always false for CC>3.
     C.CCMask = CC < 4 ? 1 << (3 - CC) : 0;
   else if (Cond == ISD::SETNE)
     // ...and the inverse of that.
     C.CCMask = CC < 4 ? ~(1 << (3 - CC)) : -1;
   else if (Cond == ISD::SETLT || Cond == ISD::SETULT)
     // bits above bit 3 for CC==0 (always false), bits above bit 0 for CC==3,
     // always true for CC>3.
     C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1;
   else if (Cond == ISD::SETGE || Cond == ISD::SETUGE)
     // ...and the inverse of that.
     C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0;
   else if (Cond == ISD::SETLE || Cond == ISD::SETULE)
     // bit 3 and above for CC==0, bit 0 and above for CC==3 (always true),
     // always true for CC>3.
     C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1;
   else if (Cond == ISD::SETGT || Cond == ISD::SETUGT)
     // ...and the inverse of that.
     C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0;
   else
     llvm_unreachable("Unexpected integer comparison type");
   C.CCMask &= CCValid;
   return C;
 }
 
 // Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1.
 static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1,
                          ISD::CondCode Cond, const SDLoc &DL,
                          SDValue Chain = SDValue(),
                          bool IsSignaling = false) {
   if (CmpOp1.getOpcode() == ISD::Constant) {
     assert(!Chain);
     uint64_t Constant = cast<ConstantSDNode>(CmpOp1)->getZExtValue();
     unsigned Opcode, CCValid;
     if (CmpOp0.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
         CmpOp0.getResNo() == 0 && CmpOp0->hasNUsesOfValue(1, 0) &&
         isIntrinsicWithCCAndChain(CmpOp0, Opcode, CCValid))
       return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid, Constant, Cond);
     if (CmpOp0.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
         CmpOp0.getResNo() == CmpOp0->getNumValues() - 1 &&
         isIntrinsicWithCC(CmpOp0, Opcode, CCValid))
       return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid, Constant, Cond);
   }
   Comparison C(CmpOp0, CmpOp1, Chain);
   C.CCMask = CCMaskForCondCode(Cond);
   if (C.Op0.getValueType().isFloatingPoint()) {
     C.CCValid = SystemZ::CCMASK_FCMP;
     if (!C.Chain)
       C.Opcode = SystemZISD::FCMP;
     else if (!IsSignaling)
       C.Opcode = SystemZISD::STRICT_FCMP;
     else
       C.Opcode = SystemZISD::STRICT_FCMPS;
     adjustForFNeg(C);
   } else {
     assert(!C.Chain);
     C.CCValid = SystemZ::CCMASK_ICMP;
     C.Opcode = SystemZISD::ICMP;
     // Choose the type of comparison.  Equality and inequality tests can
     // use either signed or unsigned comparisons.  The choice also doesn't
     // matter if both sign bits are known to be clear.  In those cases we
     // want to give the main isel code the freedom to choose whichever
     // form fits best.
     if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
         C.CCMask == SystemZ::CCMASK_CMP_NE ||
         (DAG.SignBitIsZero(C.Op0) && DAG.SignBitIsZero(C.Op1)))
       C.ICmpType = SystemZICMP::Any;
     else if (C.CCMask & SystemZ::CCMASK_CMP_UO)
       C.ICmpType = SystemZICMP::UnsignedOnly;
     else
       C.ICmpType = SystemZICMP::SignedOnly;
     C.CCMask &= ~SystemZ::CCMASK_CMP_UO;
     adjustForRedundantAnd(DAG, DL, C);
     adjustZeroCmp(DAG, DL, C);
     adjustSubwordCmp(DAG, DL, C);
     adjustForSubtraction(DAG, DL, C);
     adjustForLTGFR(C);
     adjustICmpTruncate(DAG, DL, C);
   }
 
   if (shouldSwapCmpOperands(C)) {
     std::swap(C.Op0, C.Op1);
     C.CCMask = SystemZ::reverseCCMask(C.CCMask);
   }
 
   adjustForTestUnderMask(DAG, DL, C);
   return C;
 }
 
 // Emit the comparison instruction described by C.
 static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
   if (!C.Op1.getNode()) {
     SDNode *Node;
     switch (C.Op0.getOpcode()) {
     case ISD::INTRINSIC_W_CHAIN:
       Node = emitIntrinsicWithCCAndChain(DAG, C.Op0, C.Opcode);
       return SDValue(Node, 0);
     case ISD::INTRINSIC_WO_CHAIN:
       Node = emitIntrinsicWithCC(DAG, C.Op0, C.Opcode);
       return SDValue(Node, Node->getNumValues() - 1);
     default:
       llvm_unreachable("Invalid comparison operands");
     }
   }
   if (C.Opcode == SystemZISD::ICMP)
     return DAG.getNode(SystemZISD::ICMP, DL, MVT::i32, C.Op0, C.Op1,
                        DAG.getTargetConstant(C.ICmpType, DL, MVT::i32));
   if (C.Opcode == SystemZISD::TM) {
     bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) !=
                          bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1));
     return DAG.getNode(SystemZISD::TM, DL, MVT::i32, C.Op0, C.Op1,
                        DAG.getTargetConstant(RegisterOnly, DL, MVT::i32));
   }
   if (C.Chain) {
     SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
     return DAG.getNode(C.Opcode, DL, VTs, C.Chain, C.Op0, C.Op1);
   }
   return DAG.getNode(C.Opcode, DL, MVT::i32, C.Op0, C.Op1);
 }
 
 // Implement a 32-bit *MUL_LOHI operation by extending both operands to
 // 64 bits.  Extend is the extension type to use.  Store the high part
 // in Hi and the low part in Lo.
 static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend,
                             SDValue Op0, SDValue Op1, SDValue &Hi,
                             SDValue &Lo) {
   Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0);
   Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1);
   SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1);
   Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
                    DAG.getConstant(32, DL, MVT::i64));
   Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi);
   Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul);
 }
 
 // Lower a binary operation that produces two VT results, one in each
 // half of a GR128 pair.  Op0 and Op1 are the VT operands to the operation,
 // and Opcode performs the GR128 operation.  Store the even register result
 // in Even and the odd register result in Odd.
 static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
                              unsigned Opcode, SDValue Op0, SDValue Op1,
                              SDValue &Even, SDValue &Odd) {
   SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, Op0, Op1);
   bool Is32Bit = is32Bit(VT);
   Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result);
   Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result);
 }
 
 // Return an i32 value that is 1 if the CC value produced by CCReg is
 // in the mask CCMask and 0 otherwise.  CC is known to have a value
 // in CCValid, so other values can be ignored.
 static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg,
                          unsigned CCValid, unsigned CCMask) {
   SDValue Ops[] = {DAG.getConstant(1, DL, MVT::i32),
                    DAG.getConstant(0, DL, MVT::i32),
                    DAG.getTargetConstant(CCValid, DL, MVT::i32),
                    DAG.getTargetConstant(CCMask, DL, MVT::i32), CCReg};
   return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, MVT::i32, Ops);
 }
 
 // Return the SystemISD vector comparison operation for CC, or 0 if it cannot
 // be done directly.  Mode is CmpMode::Int for integer comparisons, CmpMode::FP
 // for regular floating-point comparisons, CmpMode::StrictFP for strict (quiet)
 // floating-point comparisons, and CmpMode::SignalingFP for strict signaling
 // floating-point comparisons.
 enum class CmpMode { Int, FP, StrictFP, SignalingFP };
 static unsigned getVectorComparison(ISD::CondCode CC, CmpMode Mode) {
   switch (CC) {
   case ISD::SETOEQ:
   case ISD::SETEQ:
     switch (Mode) {
     case CmpMode::Int:         return SystemZISD::VICMPE;
     case CmpMode::FP:          return SystemZISD::VFCMPE;
     case CmpMode::StrictFP:    return SystemZISD::STRICT_VFCMPE;
     case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPES;
     }
     llvm_unreachable("Bad mode");
 
   case ISD::SETOGE:
   case ISD::SETGE:
     switch (Mode) {
     case CmpMode::Int:         return 0;
     case CmpMode::FP:          return SystemZISD::VFCMPHE;
     case CmpMode::StrictFP:    return SystemZISD::STRICT_VFCMPHE;
     case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHES;
     }
     llvm_unreachable("Bad mode");
 
   case ISD::SETOGT:
   case ISD::SETGT:
     switch (Mode) {
     case CmpMode::Int:         return SystemZISD::VICMPH;
     case CmpMode::FP:          return SystemZISD::VFCMPH;
     case CmpMode::StrictFP:    return SystemZISD::STRICT_VFCMPH;
     case CmpMode::SignalingFP: return SystemZISD::STRICT_VFCMPHS;
     }
     llvm_unreachable("Bad mode");
 
   case ISD::SETUGT:
     switch (Mode) {
     case CmpMode::Int:         return SystemZISD::VICMPHL;
     case CmpMode::FP:          return 0;
     case CmpMode::StrictFP:    return 0;
     case CmpMode::SignalingFP: return 0;
     }
     llvm_unreachable("Bad mode");
 
   default:
     return 0;
   }
 }
 
 // Return the SystemZISD vector comparison operation for CC or its inverse,
 // or 0 if neither can be done directly.  Indicate in Invert whether the
 // result is for the inverse of CC.  Mode is as above.
 static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, CmpMode Mode,
                                             bool &Invert) {
   if (unsigned Opcode = getVectorComparison(CC, Mode)) {
     Invert = false;
     return Opcode;
   }
 
   CC = ISD::getSetCCInverse(CC, Mode == CmpMode::Int ? MVT::i32 : MVT::f32);
   if (unsigned Opcode = getVectorComparison(CC, Mode)) {
     Invert = true;
     return Opcode;
   }
 
   return 0;
 }
 
 // Return a v2f64 that contains the extended form of elements Start and Start+1
 // of v4f32 value Op.  If Chain is nonnull, return the strict form.
 static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL,
                                   SDValue Op, SDValue Chain) {
   int Mask[] = { Start, -1, Start + 1, -1 };
   Op = DAG.getVectorShuffle(MVT::v4f32, DL, Op, DAG.getUNDEF(MVT::v4f32), Mask);
   if (Chain) {
     SDVTList VTs = DAG.getVTList(MVT::v2f64, MVT::Other);
     return DAG.getNode(SystemZISD::STRICT_VEXTEND, DL, VTs, Chain, Op);
   }
   return DAG.getNode(SystemZISD::VEXTEND, DL, MVT::v2f64, Op);
 }
 
 // Build a comparison of vectors CmpOp0 and CmpOp1 using opcode Opcode,
 // producing a result of type VT.  If Chain is nonnull, return the strict form.
 SDValue SystemZTargetLowering::getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
                                             const SDLoc &DL, EVT VT,
                                             SDValue CmpOp0,
                                             SDValue CmpOp1,
                                             SDValue Chain) const {
   // There is no hardware support for v4f32 (unless we have the vector
   // enhancements facility 1), so extend the vector into two v2f64s
   // and compare those.
   if (CmpOp0.getValueType() == MVT::v4f32 &&
       !Subtarget.hasVectorEnhancements1()) {
     SDValue H0 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp0, Chain);
     SDValue L0 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp0, Chain);
     SDValue H1 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp1, Chain);
     SDValue L1 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp1, Chain);
     if (Chain) {
       SDVTList VTs = DAG.getVTList(MVT::v2i64, MVT::Other);
       SDValue HRes = DAG.getNode(Opcode, DL, VTs, Chain, H0, H1);
       SDValue LRes = DAG.getNode(Opcode, DL, VTs, Chain, L0, L1);
       SDValue Res = DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes);
       SDValue Chains[6] = { H0.getValue(1), L0.getValue(1),
                             H1.getValue(1), L1.getValue(1),
                             HRes.getValue(1), LRes.getValue(1) };
       SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
       SDValue Ops[2] = { Res, NewChain };
       return DAG.getMergeValues(Ops, DL);
     }
     SDValue HRes = DAG.getNode(Opcode, DL, MVT::v2i64, H0, H1);
     SDValue LRes = DAG.getNode(Opcode, DL, MVT::v2i64, L0, L1);
     return DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes);
   }
   if (Chain) {
     SDVTList VTs = DAG.getVTList(VT, MVT::Other);
     return DAG.getNode(Opcode, DL, VTs, Chain, CmpOp0, CmpOp1);
   }
   return DAG.getNode(Opcode, DL, VT, CmpOp0, CmpOp1);
 }
 
 // Lower a vector comparison of type CC between CmpOp0 and CmpOp1, producing
 // an integer mask of type VT.  If Chain is nonnull, we have a strict
 // floating-point comparison.  If in addition IsSignaling is true, we have
 // a strict signaling floating-point comparison.
 SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG,
                                                 const SDLoc &DL, EVT VT,
                                                 ISD::CondCode CC,
                                                 SDValue CmpOp0,
                                                 SDValue CmpOp1,
                                                 SDValue Chain,
                                                 bool IsSignaling) const {
   bool IsFP = CmpOp0.getValueType().isFloatingPoint();
   assert (!Chain || IsFP);
   assert (!IsSignaling || Chain);
   CmpMode Mode = IsSignaling ? CmpMode::SignalingFP :
                  Chain ? CmpMode::StrictFP : IsFP ? CmpMode::FP : CmpMode::Int;
   bool Invert = false;
   SDValue Cmp;
   switch (CC) {
     // Handle tests for order using (or (ogt y x) (oge x y)).
   case ISD::SETUO:
     Invert = true;
     LLVM_FALLTHROUGH;
   case ISD::SETO: {
     assert(IsFP && "Unexpected integer comparison");
     SDValue LT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode),
                               DL, VT, CmpOp1, CmpOp0, Chain);
     SDValue GE = getVectorCmp(DAG, getVectorComparison(ISD::SETOGE, Mode),
                               DL, VT, CmpOp0, CmpOp1, Chain);
     Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GE);
     if (Chain)
       Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
                           LT.getValue(1), GE.getValue(1));
     break;
   }
 
     // Handle <> tests using (or (ogt y x) (ogt x y)).
   case ISD::SETUEQ:
     Invert = true;
     LLVM_FALLTHROUGH;
   case ISD::SETONE: {
     assert(IsFP && "Unexpected integer comparison");
     SDValue LT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode),
                               DL, VT, CmpOp1, CmpOp0, Chain);
     SDValue GT = getVectorCmp(DAG, getVectorComparison(ISD::SETOGT, Mode),
                               DL, VT, CmpOp0, CmpOp1, Chain);
     Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GT);
     if (Chain)
       Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
                           LT.getValue(1), GT.getValue(1));
     break;
   }
 
     // Otherwise a single comparison is enough.  It doesn't really
     // matter whether we try the inversion or the swap first, since
     // there are no cases where both work.
   default:
     if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
       Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1, Chain);
     else {
       CC = ISD::getSetCCSwappedOperands(CC);
       if (unsigned Opcode = getVectorComparisonOrInvert(CC, Mode, Invert))
         Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp1, CmpOp0, Chain);
       else
         llvm_unreachable("Unhandled comparison");
     }
     if (Chain)
       Chain = Cmp.getValue(1);
     break;
   }
   if (Invert) {
     SDValue Mask =
       DAG.getSplatBuildVector(VT, DL, DAG.getConstant(-1, DL, MVT::i64));
     Cmp = DAG.getNode(ISD::XOR, DL, VT, Cmp, Mask);
   }
   if (Chain && Chain.getNode() != Cmp.getNode()) {
     SDValue Ops[2] = { Cmp, Chain };
     Cmp = DAG.getMergeValues(Ops, DL);
   }
   return Cmp;
 }
 
 SDValue SystemZTargetLowering::lowerSETCC(SDValue Op,
                                           SelectionDAG &DAG) const {
   SDValue CmpOp0   = Op.getOperand(0);
   SDValue CmpOp1   = Op.getOperand(1);
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
   SDLoc DL(Op);
   EVT VT = Op.getValueType();
   if (VT.isVector())
     return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1);
 
   Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
   SDValue CCReg = emitCmp(DAG, DL, C);
   return emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask);
 }
 
 SDValue SystemZTargetLowering::lowerSTRICT_FSETCC(SDValue Op,
                                                   SelectionDAG &DAG,
                                                   bool IsSignaling) const {
   SDValue Chain    = Op.getOperand(0);
   SDValue CmpOp0   = Op.getOperand(1);
   SDValue CmpOp1   = Op.getOperand(2);
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(3))->get();
   SDLoc DL(Op);
   EVT VT = Op.getNode()->getValueType(0);
   if (VT.isVector()) {
     SDValue Res = lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1,
                                    Chain, IsSignaling);
     return Res.getValue(Op.getResNo());
   }
 
   Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL, Chain, IsSignaling));
   SDValue CCReg = emitCmp(DAG, DL, C);
   CCReg->setFlags(Op->getFlags());
   SDValue Result = emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask);
   SDValue Ops[2] = { Result, CCReg.getValue(1) };
   return DAG.getMergeValues(Ops, DL);
 }
 
 SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
   SDValue CmpOp0   = Op.getOperand(2);
   SDValue CmpOp1   = Op.getOperand(3);
   SDValue Dest     = Op.getOperand(4);
   SDLoc DL(Op);
 
   Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
   SDValue CCReg = emitCmp(DAG, DL, C);
   return DAG.getNode(
       SystemZISD::BR_CCMASK, DL, Op.getValueType(), Op.getOperand(0),
       DAG.getTargetConstant(C.CCValid, DL, MVT::i32),
       DAG.getTargetConstant(C.CCMask, DL, MVT::i32), Dest, CCReg);
 }
 
 // Return true if Pos is CmpOp and Neg is the negative of CmpOp,
 // allowing Pos and Neg to be wider than CmpOp.
 static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) {
   return (Neg.getOpcode() == ISD::SUB &&
           Neg.getOperand(0).getOpcode() == ISD::Constant &&
           cast<ConstantSDNode>(Neg.getOperand(0))->getZExtValue() == 0 &&
           Neg.getOperand(1) == Pos &&
           (Pos == CmpOp ||
            (Pos.getOpcode() == ISD::SIGN_EXTEND &&
             Pos.getOperand(0) == CmpOp)));
 }
 
 // Return the absolute or negative absolute of Op; IsNegative decides which.
 static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op,
                            bool IsNegative) {
   Op = DAG.getNode(ISD::ABS, DL, Op.getValueType(), Op);
   if (IsNegative)
     Op = DAG.getNode(ISD::SUB, DL, Op.getValueType(),
                      DAG.getConstant(0, DL, Op.getValueType()), Op);
   return Op;
 }
 
 SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
                                               SelectionDAG &DAG) const {
   SDValue CmpOp0   = Op.getOperand(0);
   SDValue CmpOp1   = Op.getOperand(1);
   SDValue TrueOp   = Op.getOperand(2);
   SDValue FalseOp  = Op.getOperand(3);
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
   SDLoc DL(Op);
 
   Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
 
   // Check for absolute and negative-absolute selections, including those
   // where the comparison value is sign-extended (for LPGFR and LNGFR).
   // This check supplements the one in DAGCombiner.
   if (C.Opcode == SystemZISD::ICMP &&
       C.CCMask != SystemZ::CCMASK_CMP_EQ &&
       C.CCMask != SystemZ::CCMASK_CMP_NE &&
       C.Op1.getOpcode() == ISD::Constant &&
       cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) {
     if (isAbsolute(C.Op0, TrueOp, FalseOp))
       return getAbsolute(DAG, DL, TrueOp, C.CCMask & SystemZ::CCMASK_CMP_LT);
     if (isAbsolute(C.Op0, FalseOp, TrueOp))
       return getAbsolute(DAG, DL, FalseOp, C.CCMask & SystemZ::CCMASK_CMP_GT);
   }
 
   SDValue CCReg = emitCmp(DAG, DL, C);
   SDValue Ops[] = {TrueOp, FalseOp,
                    DAG.getTargetConstant(C.CCValid, DL, MVT::i32),
                    DAG.getTargetConstant(C.CCMask, DL, MVT::i32), CCReg};
 
   return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, Op.getValueType(), Ops);
 }
 
 SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node,
                                                   SelectionDAG &DAG) const {
   SDLoc DL(Node);
   const GlobalValue *GV = Node->getGlobal();
   int64_t Offset = Node->getOffset();
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
   CodeModel::Model CM = DAG.getTarget().getCodeModel();
 
   SDValue Result;
   if (Subtarget.isPC32DBLSymbol(GV, CM)) {
     if (isInt<32>(Offset)) {
       // Assign anchors at 1<<12 byte boundaries.
       uint64_t Anchor = Offset & ~uint64_t(0xfff);
       Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor);
       Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
 
       // The offset can be folded into the address if it is aligned to a
       // halfword.
       Offset -= Anchor;
       if (Offset != 0 && (Offset & 1) == 0) {
         SDValue Full =
           DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor + Offset);
         Result = DAG.getNode(SystemZISD::PCREL_OFFSET, DL, PtrVT, Full, Result);
         Offset = 0;
       }
     } else {
       // Conservatively load a constant offset greater than 32 bits into a
       // register below.
       Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT);
       Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
     }
   } else {
     Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT);
     Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
     Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
                          MachinePointerInfo::getGOT(DAG.getMachineFunction()));
   }
 
   // If there was a non-zero offset that we didn't fold, create an explicit
   // addition for it.
   if (Offset != 0)
     Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result,
                          DAG.getConstant(Offset, DL, PtrVT));
 
   return Result;
 }
 
 SDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node,
                                                  SelectionDAG &DAG,
                                                  unsigned Opcode,
                                                  SDValue GOTOffset) const {
   SDLoc DL(Node);
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDValue Chain = DAG.getEntryNode();
   SDValue Glue;
 
   if (DAG.getMachineFunction().getFunction().getCallingConv() ==
       CallingConv::GHC)
     report_fatal_error("In GHC calling convention TLS is not supported");
 
   // __tls_get_offset takes the GOT offset in %r2 and the GOT in %r12.
   SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT);
   Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R12D, GOT, Glue);
   Glue = Chain.getValue(1);
   Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R2D, GOTOffset, Glue);
   Glue = Chain.getValue(1);
 
   // The first call operand is the chain and the second is the TLS symbol.
   SmallVector<SDValue, 8> Ops;
   Ops.push_back(Chain);
   Ops.push_back(DAG.getTargetGlobalAddress(Node->getGlobal(), DL,
                                            Node->getValueType(0),
                                            0, 0));
 
   // Add argument registers to the end of the list so that they are
   // known live into the call.
   Ops.push_back(DAG.getRegister(SystemZ::R2D, PtrVT));
   Ops.push_back(DAG.getRegister(SystemZ::R12D, PtrVT));
 
   // Add a register mask operand representing the call-preserved registers.
   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
   const uint32_t *Mask =
       TRI->getCallPreservedMask(DAG.getMachineFunction(), CallingConv::C);
   assert(Mask && "Missing call preserved mask for calling convention");
   Ops.push_back(DAG.getRegisterMask(Mask));
 
   // Glue the call to the argument copies.
   Ops.push_back(Glue);
 
   // Emit the call.
   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
   Chain = DAG.getNode(Opcode, DL, NodeTys, Ops);
   Glue = Chain.getValue(1);
 
   // Copy the return value from %r2.
   return DAG.getCopyFromReg(Chain, DL, SystemZ::R2D, PtrVT, Glue);
 }
 
 SDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL,
                                                   SelectionDAG &DAG) const {
   SDValue Chain = DAG.getEntryNode();
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
   // The high part of the thread pointer is in access register 0.
   SDValue TPHi = DAG.getCopyFromReg(Chain, DL, SystemZ::A0, MVT::i32);
   TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi);
 
   // The low part of the thread pointer is in access register 1.
   SDValue TPLo = DAG.getCopyFromReg(Chain, DL, SystemZ::A1, MVT::i32);
   TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo);
 
   // Merge them into a single 64-bit address.
   SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi,
                                     DAG.getConstant(32, DL, PtrVT));
   return DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo);
 }
 
 SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
                                                      SelectionDAG &DAG) const {
   if (DAG.getTarget().useEmulatedTLS())
     return LowerToTLSEmulatedModel(Node, DAG);
   SDLoc DL(Node);
   const GlobalValue *GV = Node->getGlobal();
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
   TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
 
   if (DAG.getMachineFunction().getFunction().getCallingConv() ==
       CallingConv::GHC)
     report_fatal_error("In GHC calling convention TLS is not supported");
 
   SDValue TP = lowerThreadPointer(DL, DAG);
 
   // Get the offset of GA from the thread pointer, based on the TLS model.
   SDValue Offset;
   switch (model) {
     case TLSModel::GeneralDynamic: {
       // Load the GOT offset of the tls_index (module ID / per-symbol offset).
       SystemZConstantPoolValue *CPV =
         SystemZConstantPoolValue::Create(GV, SystemZCP::TLSGD);
 
       Offset = DAG.getConstantPool(CPV, PtrVT, Align(8));
       Offset = DAG.getLoad(
           PtrVT, DL, DAG.getEntryNode(), Offset,
           MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
 
       // Call __tls_get_offset to retrieve the offset.
       Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_GDCALL, Offset);
       break;
     }
 
     case TLSModel::LocalDynamic: {
       // Load the GOT offset of the module ID.
       SystemZConstantPoolValue *CPV =
         SystemZConstantPoolValue::Create(GV, SystemZCP::TLSLDM);
 
       Offset = DAG.getConstantPool(CPV, PtrVT, Align(8));
       Offset = DAG.getLoad(
           PtrVT, DL, DAG.getEntryNode(), Offset,
           MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
 
       // Call __tls_get_offset to retrieve the module base offset.
       Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_LDCALL, Offset);
 
       // Note: The SystemZLDCleanupPass will remove redundant computations
       // of the module base offset.  Count total number of local-dynamic
       // accesses to trigger execution of that pass.
       SystemZMachineFunctionInfo* MFI =
         DAG.getMachineFunction().getInfo<SystemZMachineFunctionInfo>();
       MFI->incNumLocalDynamicTLSAccesses();
 
       // Add the per-symbol offset.
       CPV = SystemZConstantPoolValue::Create(GV, SystemZCP::DTPOFF);
 
       SDValue DTPOffset = DAG.getConstantPool(CPV, PtrVT, Align(8));
       DTPOffset = DAG.getLoad(
           PtrVT, DL, DAG.getEntryNode(), DTPOffset,
           MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
 
       Offset = DAG.getNode(ISD::ADD, DL, PtrVT, Offset, DTPOffset);
       break;
     }
 
     case TLSModel::InitialExec: {
       // Load the offset from the GOT.
       Offset = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0,
                                           SystemZII::MO_INDNTPOFF);
       Offset = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Offset);
       Offset =
           DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Offset,
                       MachinePointerInfo::getGOT(DAG.getMachineFunction()));
       break;
     }
 
     case TLSModel::LocalExec: {
       // Force the offset into the constant pool and load it from there.
       SystemZConstantPoolValue *CPV =
         SystemZConstantPoolValue::Create(GV, SystemZCP::NTPOFF);
 
       Offset = DAG.getConstantPool(CPV, PtrVT, Align(8));
       Offset = DAG.getLoad(
           PtrVT, DL, DAG.getEntryNode(), Offset,
           MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
       break;
     }
   }
 
   // Add the base and offset together.
   return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset);
 }
 
 SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node,
                                                  SelectionDAG &DAG) const {
   SDLoc DL(Node);
   const BlockAddress *BA = Node->getBlockAddress();
   int64_t Offset = Node->getOffset();
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
   SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset);
   Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
   return Result;
 }
 
 SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT,
                                               SelectionDAG &DAG) const {
   SDLoc DL(JT);
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
   SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
 
   // Use LARL to load the address of the table.
   return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
 }
 
 SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP,
                                                  SelectionDAG &DAG) const {
   SDLoc DL(CP);
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
   SDValue Result;
   if (CP->isMachineConstantPoolEntry())
     Result =
         DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, CP->getAlign());
   else
     Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlign(),
                                        CP->getOffset());
 
   // Use LARL to load the address of the constant pool entry.
   return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result);
 }
 
 SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op,
                                               SelectionDAG &DAG) const {
   auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setFrameAddressIsTaken(true);
 
   SDLoc DL(Op);
   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
   // By definition, the frame address is the address of the back chain.  (In
   // the case of packed stack without backchain, return the address where the
   // backchain would have been stored. This will either be an unused space or
   // contain a saved register).
   int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);
   SDValue BackChain = DAG.getFrameIndex(BackChainIdx, PtrVT);
 
   // FIXME The frontend should detect this case.
   if (Depth > 0) {
     report_fatal_error("Unsupported stack frame traversal count");
   }
 
   return BackChain;
 }
 
 SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op,
                                                SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   MFI.setReturnAddressIsTaken(true);
 
   if (verifyReturnAddressArgumentIsConstant(Op, DAG))
     return SDValue();
 
   SDLoc DL(Op);
   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
   // FIXME The frontend should detect this case.
   if (Depth > 0) {
     report_fatal_error("Unsupported stack frame traversal count");
   }
 
   // Return R14D, which has the return address. Mark it an implicit live-in.
   Register LinkReg = MF.addLiveIn(SystemZ::R14D, &SystemZ::GR64BitRegClass);
   return DAG.getCopyFromReg(DAG.getEntryNode(), DL, LinkReg, PtrVT);
 }
 
 SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op,
                                             SelectionDAG &DAG) const {
   SDLoc DL(Op);
   SDValue In = Op.getOperand(0);
   EVT InVT = In.getValueType();
   EVT ResVT = Op.getValueType();
 
   // Convert loads directly.  This is normally done by DAGCombiner,
   // but we need this case for bitcasts that are created during lowering
   // and which are then lowered themselves.
   if (auto *LoadN = dyn_cast<LoadSDNode>(In))
     if (ISD::isNormalLoad(LoadN)) {
       SDValue NewLoad = DAG.getLoad(ResVT, DL, LoadN->getChain(),
                                     LoadN->getBasePtr(), LoadN->getMemOperand());
       // Update the chain uses.
       DAG.ReplaceAllUsesOfValueWith(SDValue(LoadN, 1), NewLoad.getValue(1));
       return NewLoad;
     }
 
   if (InVT == MVT::i32 && ResVT == MVT::f32) {
     SDValue In64;
     if (Subtarget.hasHighWord()) {
       SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL,
                                        MVT::i64);
       In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
                                        MVT::i64, SDValue(U64, 0), In);
     } else {
       In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In);
       In64 = DAG.getNode(ISD::SHL, DL, MVT::i64, In64,
                          DAG.getConstant(32, DL, MVT::i64));
     }
     SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, In64);
     return DAG.getTargetExtractSubreg(SystemZ::subreg_h32,
                                       DL, MVT::f32, Out64);
   }
   if (InVT == MVT::f32 && ResVT == MVT::i32) {
     SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64);
     SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
                                              MVT::f64, SDValue(U64, 0), In);
     SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64);
     if (Subtarget.hasHighWord())
       return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, DL,
                                         MVT::i32, Out64);
     SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64,
                                 DAG.getConstant(32, DL, MVT::i64));
     return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift);
   }
   llvm_unreachable("Unexpected bitcast combination");
 }
 
 SDValue SystemZTargetLowering::lowerVASTART(SDValue Op,
                                             SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
   SystemZMachineFunctionInfo *FuncInfo =
     MF.getInfo<SystemZMachineFunctionInfo>();
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
   SDValue Chain   = Op.getOperand(0);
   SDValue Addr    = Op.getOperand(1);
   const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
   SDLoc DL(Op);
 
   // The initial values of each field.
   const unsigned NumFields = 4;
   SDValue Fields[NumFields] = {
     DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), DL, PtrVT),
     DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), DL, PtrVT),
     DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT),
     DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT)
   };
 
   // Store each field into its respective slot.
   SDValue MemOps[NumFields];
   unsigned Offset = 0;
   for (unsigned I = 0; I < NumFields; ++I) {
     SDValue FieldAddr = Addr;
     if (Offset != 0)
       FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr,
                               DAG.getIntPtrConstant(Offset, DL));
     MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr,
                              MachinePointerInfo(SV, Offset));
     Offset += 8;
   }
   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
 }
 
 SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op,
                                            SelectionDAG &DAG) const {
   SDValue Chain      = Op.getOperand(0);
   SDValue DstPtr     = Op.getOperand(1);
   SDValue SrcPtr     = Op.getOperand(2);
   const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
   const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
   SDLoc DL(Op);
 
   return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32, DL),
                        Align(8), /*isVolatile*/ false, /*AlwaysInline*/ false,
                        /*isTailCall*/ false, MachinePointerInfo(DstSV),
                        MachinePointerInfo(SrcSV));
 }
 
 SDValue SystemZTargetLowering::
 lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const {
   const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
   MachineFunction &MF = DAG.getMachineFunction();
   bool RealignOpt = !MF.getFunction().hasFnAttribute("no-realign-stack");
   bool StoreBackchain = MF.getFunction().hasFnAttribute("backchain");
 
   SDValue Chain = Op.getOperand(0);
   SDValue Size  = Op.getOperand(1);
   SDValue Align = Op.getOperand(2);
   SDLoc DL(Op);
 
   // If user has set the no alignment function attribute, ignore
   // alloca alignments.
   uint64_t AlignVal =
       (RealignOpt ? cast<ConstantSDNode>(Align)->getZExtValue() : 0);
 
   uint64_t StackAlign = TFI->getStackAlignment();
   uint64_t RequiredAlign = std::max(AlignVal, StackAlign);
   uint64_t ExtraAlignSpace = RequiredAlign - StackAlign;
 
   Register SPReg = getStackPointerRegisterToSaveRestore();
   SDValue NeededSpace = Size;
 
   // Get a reference to the stack pointer.
   SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64);
 
   // If we need a backchain, save it now.
   SDValue Backchain;
   if (StoreBackchain)
     Backchain = DAG.getLoad(MVT::i64, DL, Chain, getBackchainAddress(OldSP, DAG),
                             MachinePointerInfo());
 
   // Add extra space for alignment if needed.
   if (ExtraAlignSpace)
     NeededSpace = DAG.getNode(ISD::ADD, DL, MVT::i64, NeededSpace,
                               DAG.getConstant(ExtraAlignSpace, DL, MVT::i64));
 
   // Get the new stack pointer value.
   SDValue NewSP;
   if (hasInlineStackProbe(MF)) {
     NewSP = DAG.getNode(SystemZISD::PROBED_ALLOCA, DL,
                 DAG.getVTList(MVT::i64, MVT::Other), Chain, OldSP, NeededSpace);
     Chain = NewSP.getValue(1);
   }
   else {
     NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, NeededSpace);
     // Copy the new stack pointer back.
     Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP);
   }
 
   // The allocated data lives above the 160 bytes allocated for the standard
   // frame, plus any outgoing stack arguments.  We don't know how much that
   // amounts to yet, so emit a special ADJDYNALLOC placeholder.
   SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64);
   SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust);
 
   // Dynamically realign if needed.
   if (RequiredAlign > StackAlign) {
     Result =
       DAG.getNode(ISD::ADD, DL, MVT::i64, Result,
                   DAG.getConstant(ExtraAlignSpace, DL, MVT::i64));
     Result =
       DAG.getNode(ISD::AND, DL, MVT::i64, Result,
                   DAG.getConstant(~(RequiredAlign - 1), DL, MVT::i64));
   }
 
   if (StoreBackchain)
     Chain = DAG.getStore(Chain, DL, Backchain, getBackchainAddress(NewSP, DAG),
                          MachinePointerInfo());
 
   SDValue Ops[2] = { Result, Chain };
   return DAG.getMergeValues(Ops, DL);
 }
 
 SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
     SDValue Op, SelectionDAG &DAG) const {
   SDLoc DL(Op);
 
   return DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64);
 }
 
 SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op,
                                               SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
   SDLoc DL(Op);
   SDValue Ops[2];
   if (is32Bit(VT))
     // Just do a normal 64-bit multiplication and extract the results.
     // We define this so that it can be used for constant division.
     lowerMUL_LOHI32(DAG, DL, ISD::SIGN_EXTEND, Op.getOperand(0),
                     Op.getOperand(1), Ops[1], Ops[0]);
   else if (Subtarget.hasMiscellaneousExtensions2())
     // SystemZISD::SMUL_LOHI returns the low result in the odd register and
     // the high result in the even register.  ISD::SMUL_LOHI is defined to
     // return the low half first, so the results are in reverse order.
     lowerGR128Binary(DAG, DL, VT, SystemZISD::SMUL_LOHI,
                      Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]);
   else {
     // Do a full 128-bit multiplication based on SystemZISD::UMUL_LOHI:
     //
     //   (ll * rl) + ((lh * rl) << 64) + ((ll * rh) << 64)
     //
     // but using the fact that the upper halves are either all zeros
     // or all ones:
     //
     //   (ll * rl) - ((lh & rl) << 64) - ((ll & rh) << 64)
     //
     // and grouping the right terms together since they are quicker than the
     // multiplication:
     //
     //   (ll * rl) - (((lh & rl) + (ll & rh)) << 64)
     SDValue C63 = DAG.getConstant(63, DL, MVT::i64);
     SDValue LL = Op.getOperand(0);
     SDValue RL = Op.getOperand(1);
     SDValue LH = DAG.getNode(ISD::SRA, DL, VT, LL, C63);
     SDValue RH = DAG.getNode(ISD::SRA, DL, VT, RL, C63);
     // SystemZISD::UMUL_LOHI returns the low result in the odd register and
     // the high result in the even register.  ISD::SMUL_LOHI is defined to
     // return the low half first, so the results are in reverse order.
     lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI,
                      LL, RL, Ops[1], Ops[0]);
     SDValue NegLLTimesRH = DAG.getNode(ISD::AND, DL, VT, LL, RH);
     SDValue NegLHTimesRL = DAG.getNode(ISD::AND, DL, VT, LH, RL);
     SDValue NegSum = DAG.getNode(ISD::ADD, DL, VT, NegLLTimesRH, NegLHTimesRL);
     Ops[1] = DAG.getNode(ISD::SUB, DL, VT, Ops[1], NegSum);
   }
   return DAG.getMergeValues(Ops, DL);
 }
 
 SDValue SystemZTargetLowering::lowerUMUL_LOHI(SDValue Op,
                                               SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
   SDLoc DL(Op);
   SDValue Ops[2];
   if (is32Bit(VT))
     // Just do a normal 64-bit multiplication and extract the results.
     // We define this so that it can be used for constant division.
     lowerMUL_LOHI32(DAG, DL, ISD::ZERO_EXTEND, Op.getOperand(0),
                     Op.getOperand(1), Ops[1], Ops[0]);
   else
     // SystemZISD::UMUL_LOHI returns the low result in the odd register and
     // the high result in the even register.  ISD::UMUL_LOHI is defined to
     // return the low half first, so the results are in reverse order.
     lowerGR128Binary(DAG, DL, VT, SystemZISD::UMUL_LOHI,
                      Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]);
   return DAG.getMergeValues(Ops, DL);
 }
 
 SDValue SystemZTargetLowering::lowerSDIVREM(SDValue Op,
                                             SelectionDAG &DAG) const {
   SDValue Op0 = Op.getOperand(0);
   SDValue Op1 = Op.getOperand(1);
   EVT VT = Op.getValueType();
   SDLoc DL(Op);
 
   // We use DSGF for 32-bit division.  This means the first operand must
   // always be 64-bit, and the second operand should be 32-bit whenever
   // that is possible, to improve performance.
   if (is32Bit(VT))
     Op0 = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op0);
   else if (DAG.ComputeNumSignBits(Op1) > 32)
     Op1 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1);
 
   // DSG(F) returns the remainder in the even register and the
   // quotient in the odd register.
   SDValue Ops[2];
   lowerGR128Binary(DAG, DL, VT, SystemZISD::SDIVREM, Op0, Op1, Ops[1], Ops[0]);
   return DAG.getMergeValues(Ops, DL);
 }
 
 SDValue SystemZTargetLowering::lowerUDIVREM(SDValue Op,
                                             SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
   SDLoc DL(Op);
 
   // DL(G) returns the remainder in the even register and the
   // quotient in the odd register.
   SDValue Ops[2];
   lowerGR128Binary(DAG, DL, VT, SystemZISD::UDIVREM,
                    Op.getOperand(0), Op.getOperand(1), Ops[1], Ops[0]);
   return DAG.getMergeValues(Ops, DL);
 }
 
 SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
   assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation");
 
   // Get the known-zero masks for each operand.
   SDValue Ops[] = {Op.getOperand(0), Op.getOperand(1)};
   KnownBits Known[2] = {DAG.computeKnownBits(Ops[0]),
                         DAG.computeKnownBits(Ops[1])};
 
   // See if the upper 32 bits of one operand and the lower 32 bits of the
   // other are known zero.  They are the low and high operands respectively.
   uint64_t Masks[] = { Known[0].Zero.getZExtValue(),
                        Known[1].Zero.getZExtValue() };
   unsigned High, Low;
   if ((Masks[0] >> 32) == 0xffffffff && uint32_t(Masks[1]) == 0xffffffff)
     High = 1, Low = 0;
   else if ((Masks[1] >> 32) == 0xffffffff && uint32_t(Masks[0]) == 0xffffffff)
     High = 0, Low = 1;
   else
     return Op;
 
   SDValue LowOp = Ops[Low];
   SDValue HighOp = Ops[High];
 
   // If the high part is a constant, we're better off using IILH.
   if (HighOp.getOpcode() == ISD::Constant)
     return Op;
 
   // If the low part is a constant that is outside the range of LHI,
   // then we're better off using IILF.
   if (LowOp.getOpcode() == ISD::Constant) {
     int64_t Value = int32_t(cast<ConstantSDNode>(LowOp)->getZExtValue());
     if (!isInt<16>(Value))
       return Op;
   }
 
   // Check whether the high part is an AND that doesn't change the
   // high 32 bits and just masks out low bits.  We can skip it if so.
   if (HighOp.getOpcode() == ISD::AND &&
       HighOp.getOperand(1).getOpcode() == ISD::Constant) {
     SDValue HighOp0 = HighOp.getOperand(0);
     uint64_t Mask = cast<ConstantSDNode>(HighOp.getOperand(1))->getZExtValue();
     if (DAG.MaskedValueIsZero(HighOp0, APInt(64, ~(Mask | 0xffffffff))))
       HighOp = HighOp0;
   }
 
   // Take advantage of the fact that all GR32 operations only change the
   // low 32 bits by truncating Low to an i32 and inserting it directly
   // using a subreg.  The interesting cases are those where the truncation
   // can be folded.
   SDLoc DL(Op);
   SDValue Low32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LowOp);
   return DAG.getTargetInsertSubreg(SystemZ::subreg_l32, DL,
                                    MVT::i64, HighOp, Low32);
 }
 
 // Lower SADDO/SSUBO/UADDO/USUBO nodes.
 SDValue SystemZTargetLowering::lowerXALUO(SDValue Op,
                                           SelectionDAG &DAG) const {
   SDNode *N = Op.getNode();
   SDValue LHS = N->getOperand(0);
   SDValue RHS = N->getOperand(1);
   SDLoc DL(N);
   unsigned BaseOp = 0;
   unsigned CCValid = 0;
   unsigned CCMask = 0;
 
   switch (Op.getOpcode()) {
   default: llvm_unreachable("Unknown instruction!");
   case ISD::SADDO:
     BaseOp = SystemZISD::SADDO;
     CCValid = SystemZ::CCMASK_ARITH;
     CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
     break;
   case ISD::SSUBO:
     BaseOp = SystemZISD::SSUBO;
     CCValid = SystemZ::CCMASK_ARITH;
     CCMask = SystemZ::CCMASK_ARITH_OVERFLOW;
     break;
   case ISD::UADDO:
     BaseOp = SystemZISD::UADDO;
     CCValid = SystemZ::CCMASK_LOGICAL;
     CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
     break;
   case ISD::USUBO:
     BaseOp = SystemZISD::USUBO;
     CCValid = SystemZ::CCMASK_LOGICAL;
     CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
     break;
   }
 
   SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32);
   SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
 
   SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask);
   if (N->getValueType(1) == MVT::i1)
     SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
 
   return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC);
 }
 
 static bool isAddCarryChain(SDValue Carry) {
   while (Carry.getOpcode() == ISD::ADDCARRY)
     Carry = Carry.getOperand(2);
   return Carry.getOpcode() == ISD::UADDO;
 }
 
 static bool isSubBorrowChain(SDValue Carry) {
   while (Carry.getOpcode() == ISD::SUBCARRY)
     Carry = Carry.getOperand(2);
   return Carry.getOpcode() == ISD::USUBO;
 }
 
 // Lower ADDCARRY/SUBCARRY nodes.
 SDValue SystemZTargetLowering::lowerADDSUBCARRY(SDValue Op,
                                                 SelectionDAG &DAG) const {
 
   SDNode *N = Op.getNode();
   MVT VT = N->getSimpleValueType(0);
 
   // Let legalize expand this if it isn't a legal type yet.
   if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
     return SDValue();
 
   SDValue LHS = N->getOperand(0);
   SDValue RHS = N->getOperand(1);
   SDValue Carry = Op.getOperand(2);
   SDLoc DL(N);
   unsigned BaseOp = 0;
   unsigned CCValid = 0;
   unsigned CCMask = 0;
 
   switch (Op.getOpcode()) {
   default: llvm_unreachable("Unknown instruction!");
   case ISD::ADDCARRY:
     if (!isAddCarryChain(Carry))
       return SDValue();
 
     BaseOp = SystemZISD::ADDCARRY;
     CCValid = SystemZ::CCMASK_LOGICAL;
     CCMask = SystemZ::CCMASK_LOGICAL_CARRY;
     break;
   case ISD::SUBCARRY:
     if (!isSubBorrowChain(Carry))
       return SDValue();
 
     BaseOp = SystemZISD::SUBCARRY;
     CCValid = SystemZ::CCMASK_LOGICAL;
     CCMask = SystemZ::CCMASK_LOGICAL_BORROW;
     break;
   }
 
   // Set the condition code from the carry flag.
   Carry = DAG.getNode(SystemZISD::GET_CCMASK, DL, MVT::i32, Carry,
                       DAG.getConstant(CCValid, DL, MVT::i32),
                       DAG.getConstant(CCMask, DL, MVT::i32));
 
   SDVTList VTs = DAG.getVTList(VT, MVT::i32);
   SDValue Result = DAG.getNode(BaseOp, DL, VTs, LHS, RHS, Carry);
 
   SDValue SetCC = emitSETCC(DAG, DL, Result.getValue(1), CCValid, CCMask);
   if (N->getValueType(1) == MVT::i1)
     SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
 
   return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, SetCC);
 }
 
 SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
                                           SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
   SDLoc DL(Op);
   Op = Op.getOperand(0);
 
   // Handle vector types via VPOPCT.
   if (VT.isVector()) {
     Op = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Op);
     Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::v16i8, Op);
     switch (VT.getScalarSizeInBits()) {
     case 8:
       break;
     case 16: {
       Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
       SDValue Shift = DAG.getConstant(8, DL, MVT::i32);
       SDValue Tmp = DAG.getNode(SystemZISD::VSHL_BY_SCALAR, DL, VT, Op, Shift);
       Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
       Op = DAG.getNode(SystemZISD::VSRL_BY_SCALAR, DL, VT, Op, Shift);
       break;
     }
     case 32: {
       SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
                                             DAG.getConstant(0, DL, MVT::i32));
       Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
       break;
     }
     case 64: {
       SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
                                             DAG.getConstant(0, DL, MVT::i32));
       Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::v4i32, Op, Tmp);
       Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
       break;
     }
     default:
       llvm_unreachable("Unexpected type");
     }
     return Op;
   }
 
   // Get the known-zero mask for the operand.
   KnownBits Known = DAG.computeKnownBits(Op);
   unsigned NumSignificantBits = Known.getMaxValue().getActiveBits();
   if (NumSignificantBits == 0)
     return DAG.getConstant(0, DL, VT);
 
   // Skip known-zero high parts of the operand.
   int64_t OrigBitSize = VT.getSizeInBits();
   int64_t BitSize = (int64_t)1 << Log2_32_Ceil(NumSignificantBits);
   BitSize = std::min(BitSize, OrigBitSize);
 
   // The POPCNT instruction counts the number of bits in each byte.
   Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op);
   Op = DAG.getNode(SystemZISD::POPCNT, DL, MVT::i64, Op);
   Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
 
   // Add up per-byte counts in a binary tree.  All bits of Op at
   // position larger than BitSize remain zero throughout.
   for (int64_t I = BitSize / 2; I >= 8; I = I / 2) {
     SDValue Tmp = DAG.getNode(ISD::SHL, DL, VT, Op, DAG.getConstant(I, DL, VT));
     if (BitSize != OrigBitSize)
       Tmp = DAG.getNode(ISD::AND, DL, VT, Tmp,
                         DAG.getConstant(((uint64_t)1 << BitSize) - 1, DL, VT));
     Op = DAG.getNode(ISD::ADD, DL, VT, Op, Tmp);
   }
 
   // Extract overall result from high byte.
   if (BitSize > 8)
     Op = DAG.getNode(ISD::SRL, DL, VT, Op,
                      DAG.getConstant(BitSize - 8, DL, VT));
 
   return Op;
 }
 
 SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
                                                  SelectionDAG &DAG) const {
   SDLoc DL(Op);
   AtomicOrdering FenceOrdering = static_cast<AtomicOrdering>(
     cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue());
   SyncScope::ID FenceSSID = static_cast<SyncScope::ID>(
     cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
 
   // The only fence that needs an instruction is a sequentially-consistent
   // cross-thread fence.
   if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
       FenceSSID == SyncScope::System) {
     return SDValue(DAG.getMachineNode(SystemZ::Serialize, DL, MVT::Other,
                                       Op.getOperand(0)),
                    0);
   }
 
   // MEMBARRIER is a compiler barrier; it codegens to a no-op.
   return DAG.getNode(SystemZISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
 }
 
 // Op is an atomic load.  Lower it into a normal volatile load.
 SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
                                                 SelectionDAG &DAG) const {
   auto *Node = cast<AtomicSDNode>(Op.getNode());
   return DAG.getExtLoad(ISD::EXTLOAD, SDLoc(Op), Op.getValueType(),
                         Node->getChain(), Node->getBasePtr(),
                         Node->getMemoryVT(), Node->getMemOperand());
 }
 
 // Op is an atomic store.  Lower it into a normal volatile store.
 SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
                                                  SelectionDAG &DAG) const {
   auto *Node = cast<AtomicSDNode>(Op.getNode());
   SDValue Chain = DAG.getTruncStore(Node->getChain(), SDLoc(Op), Node->getVal(),
                                     Node->getBasePtr(), Node->getMemoryVT(),
                                     Node->getMemOperand());
   // We have to enforce sequential consistency by performing a
   // serialization operation after the store.
   if (Node->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent)
     Chain = SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op),
                                        MVT::Other, Chain), 0);
   return Chain;
 }
 
 // Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation.  Lower the first
 // two into the fullword ATOMIC_LOADW_* operation given by Opcode.
 SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op,
                                                    SelectionDAG &DAG,
                                                    unsigned Opcode) const {
   auto *Node = cast<AtomicSDNode>(Op.getNode());
 
   // 32-bit operations need no code outside the main loop.
   EVT NarrowVT = Node->getMemoryVT();
   EVT WideVT = MVT::i32;
   if (NarrowVT == WideVT)
     return Op;
 
   int64_t BitSize = NarrowVT.getSizeInBits();
   SDValue ChainIn = Node->getChain();
   SDValue Addr = Node->getBasePtr();
   SDValue Src2 = Node->getVal();
   MachineMemOperand *MMO = Node->getMemOperand();
   SDLoc DL(Node);
   EVT PtrVT = Addr.getValueType();
 
   // Convert atomic subtracts of constants into additions.
   if (Opcode == SystemZISD::ATOMIC_LOADW_SUB)
     if (auto *Const = dyn_cast<ConstantSDNode>(Src2)) {
       Opcode = SystemZISD::ATOMIC_LOADW_ADD;
       Src2 = DAG.getConstant(-Const->getSExtValue(), DL, Src2.getValueType());
     }
 
   // Get the address of the containing word.
   SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr,
                                     DAG.getConstant(-4, DL, PtrVT));
 
   // Get the number of bits that the word must be rotated left in order
   // to bring the field to the top bits of a GR32.
   SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr,
                                  DAG.getConstant(3, DL, PtrVT));
   BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift);
 
   // Get the complementing shift amount, for rotating a field in the top
   // bits back to its proper position.
   SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT,
                                     DAG.getConstant(0, DL, WideVT), BitShift);
 
   // Extend the source operand to 32 bits and prepare it for the inner loop.
   // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other
   // operations require the source to be shifted in advance.  (This shift
   // can be folded if the source is constant.)  For AND and NAND, the lower
   // bits must be set, while for other opcodes they should be left clear.
   if (Opcode != SystemZISD::ATOMIC_SWAPW)
     Src2 = DAG.getNode(ISD::SHL, DL, WideVT, Src2,
                        DAG.getConstant(32 - BitSize, DL, WideVT));
   if (Opcode == SystemZISD::ATOMIC_LOADW_AND ||
       Opcode == SystemZISD::ATOMIC_LOADW_NAND)
     Src2 = DAG.getNode(ISD::OR, DL, WideVT, Src2,
                        DAG.getConstant(uint32_t(-1) >> BitSize, DL, WideVT));
 
   // Construct the ATOMIC_LOADW_* node.
   SDVTList VTList = DAG.getVTList(WideVT, MVT::Other);
   SDValue Ops[] = { ChainIn, AlignedAddr, Src2, BitShift, NegBitShift,
                     DAG.getConstant(BitSize, DL, WideVT) };
   SDValue AtomicOp = DAG.getMemIntrinsicNode(Opcode, DL, VTList, Ops,
                                              NarrowVT, MMO);
 
   // Rotate the result of the final CS so that the field is in the lower
   // bits of a GR32, then truncate it.
   SDValue ResultShift = DAG.getNode(ISD::ADD, DL, WideVT, BitShift,
                                     DAG.getConstant(BitSize, DL, WideVT));
   SDValue Result = DAG.getNode(ISD::ROTL, DL, WideVT, AtomicOp, ResultShift);
 
   SDValue RetOps[2] = { Result, AtomicOp.getValue(1) };
   return DAG.getMergeValues(RetOps, DL);
 }
 
 // Op is an ATOMIC_LOAD_SUB operation.  Lower 8- and 16-bit operations
 // into ATOMIC_LOADW_SUBs and decide whether to convert 32- and 64-bit
 // operations into additions.
 SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op,
                                                     SelectionDAG &DAG) const {
   auto *Node = cast<AtomicSDNode>(Op.getNode());
   EVT MemVT = Node->getMemoryVT();
   if (MemVT == MVT::i32 || MemVT == MVT::i64) {
     // A full-width operation.
     assert(Op.getValueType() == MemVT && "Mismatched VTs");
     SDValue Src2 = Node->getVal();
     SDValue NegSrc2;
     SDLoc DL(Src2);
 
     if (auto *Op2 = dyn_cast<ConstantSDNode>(Src2)) {
       // Use an addition if the operand is constant and either LAA(G) is
       // available or the negative value is in the range of A(G)FHI.
       int64_t Value = (-Op2->getAPIntValue()).getSExtValue();
       if (isInt<32>(Value) || Subtarget.hasInterlockedAccess1())
         NegSrc2 = DAG.getConstant(Value, DL, MemVT);
     } else if (Subtarget.hasInterlockedAccess1())
       // Use LAA(G) if available.
       NegSrc2 = DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT),
                             Src2);
 
     if (NegSrc2.getNode())
       return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT,
                            Node->getChain(), Node->getBasePtr(), NegSrc2,
                            Node->getMemOperand());
 
     // Use the node as-is.
     return Op;
   }
 
   return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB);
 }
 
 // Lower 8/16/32/64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS node.
 SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
                                                     SelectionDAG &DAG) const {
   auto *Node = cast<AtomicSDNode>(Op.getNode());
   SDValue ChainIn = Node->getOperand(0);
   SDValue Addr = Node->getOperand(1);
   SDValue CmpVal = Node->getOperand(2);
   SDValue SwapVal = Node->getOperand(3);
   MachineMemOperand *MMO = Node->getMemOperand();
   SDLoc DL(Node);
 
   // We have native support for 32-bit and 64-bit compare and swap, but we
   // still need to expand extracting the "success" result from the CC.
   EVT NarrowVT = Node->getMemoryVT();
   EVT WideVT = NarrowVT == MVT::i64 ? MVT::i64 : MVT::i32;
   if (NarrowVT == WideVT) {
     SDVTList Tys = DAG.getVTList(WideVT, MVT::i32, MVT::Other);
     SDValue Ops[] = { ChainIn, Addr, CmpVal, SwapVal };
     SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAP,
                                                DL, Tys, Ops, NarrowVT, MMO);
     SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
                                 SystemZ::CCMASK_CS, SystemZ::CCMASK_CS_EQ);
 
     DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), AtomicOp.getValue(0));
     DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
     DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(2));
     return SDValue();
   }
 
   // Convert 8-bit and 16-bit compare and swap to a loop, implemented
   // via a fullword ATOMIC_CMP_SWAPW operation.
   int64_t BitSize = NarrowVT.getSizeInBits();
   EVT PtrVT = Addr.getValueType();
 
   // Get the address of the containing word.
   SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr,
                                     DAG.getConstant(-4, DL, PtrVT));
 
   // Get the number of bits that the word must be rotated left in order
   // to bring the field to the top bits of a GR32.
   SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr,
                                  DAG.getConstant(3, DL, PtrVT));
   BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift);
 
   // Get the complementing shift amount, for rotating a field in the top
   // bits back to its proper position.
   SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT,
                                     DAG.getConstant(0, DL, WideVT), BitShift);
 
   // Construct the ATOMIC_CMP_SWAPW node.
   SDVTList VTList = DAG.getVTList(WideVT, MVT::i32, MVT::Other);
   SDValue Ops[] = { ChainIn, AlignedAddr, CmpVal, SwapVal, BitShift,
                     NegBitShift, DAG.getConstant(BitSize, DL, WideVT) };
   SDValue AtomicOp = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAPW, DL,
                                              VTList, Ops, NarrowVT, MMO);
   SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
                               SystemZ::CCMASK_ICMP, SystemZ::CCMASK_CMP_EQ);
 
   // emitAtomicCmpSwapW() will zero extend the result (original value).
   SDValue OrigVal = DAG.getNode(ISD::AssertZext, DL, WideVT, AtomicOp.getValue(0),
                                 DAG.getValueType(NarrowVT));
   DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), OrigVal);
   DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
   DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(2));
   return SDValue();
 }
 
 MachineMemOperand::Flags
 SystemZTargetLowering::getTargetMMOFlags(const Instruction &I) const {
   // Because of how we convert atomic_load and atomic_store to normal loads and
   // stores in the DAG, we need to ensure that the MMOs are marked volatile
   // since DAGCombine hasn't been updated to account for atomic, but non
   // volatile loads.  (See D57601)
   if (auto *SI = dyn_cast<StoreInst>(&I))
     if (SI->isAtomic())
       return MachineMemOperand::MOVolatile;
   if (auto *LI = dyn_cast<LoadInst>(&I))
     if (LI->isAtomic())
       return MachineMemOperand::MOVolatile;
   if (auto *AI = dyn_cast<AtomicRMWInst>(&I))
     if (AI->isAtomic())
       return MachineMemOperand::MOVolatile;
   if (auto *AI = dyn_cast<AtomicCmpXchgInst>(&I))
     if (AI->isAtomic())
       return MachineMemOperand::MOVolatile;
   return MachineMemOperand::MONone;
 }
 
 SDValue SystemZTargetLowering::lowerSTACKSAVE(SDValue Op,
                                               SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
   const SystemZSubtarget *Subtarget = &MF.getSubtarget<SystemZSubtarget>();
   auto *Regs = Subtarget->getSpecialRegisters();
   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     report_fatal_error("Variable-sized stack allocations are not supported "
                        "in GHC calling convention");
   return DAG.getCopyFromReg(Op.getOperand(0), SDLoc(Op),
                             Regs->getStackPointerRegister(), Op.getValueType());
 }
 
 SDValue SystemZTargetLowering::lowerSTACKRESTORE(SDValue Op,
                                                  SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
   const SystemZSubtarget *Subtarget = &MF.getSubtarget<SystemZSubtarget>();
   auto *Regs = Subtarget->getSpecialRegisters();
   bool StoreBackchain = MF.getFunction().hasFnAttribute("backchain");
 
   if (MF.getFunction().getCallingConv() == CallingConv::GHC)
     report_fatal_error("Variable-sized stack allocations are not supported "
                        "in GHC calling convention");
 
   SDValue Chain = Op.getOperand(0);
   SDValue NewSP = Op.getOperand(1);
   SDValue Backchain;
   SDLoc DL(Op);
 
   if (StoreBackchain) {
     SDValue OldSP = DAG.getCopyFromReg(
         Chain, DL, Regs->getStackPointerRegister(), MVT::i64);
     Backchain = DAG.getLoad(MVT::i64, DL, Chain, getBackchainAddress(OldSP, DAG),
                             MachinePointerInfo());
   }
 
   Chain = DAG.getCopyToReg(Chain, DL, Regs->getStackPointerRegister(), NewSP);
 
   if (StoreBackchain)
     Chain = DAG.getStore(Chain, DL, Backchain, getBackchainAddress(NewSP, DAG),
                          MachinePointerInfo());
 
   return Chain;
 }
 
 SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
                                              SelectionDAG &DAG) const {
   bool IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
   if (!IsData)
     // Just preserve the chain.
     return Op.getOperand(0);
 
   SDLoc DL(Op);
   bool IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
   unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ;
   auto *Node = cast<MemIntrinsicSDNode>(Op.getNode());
   SDValue Ops[] = {Op.getOperand(0), DAG.getTargetConstant(Code, DL, MVT::i32),
                    Op.getOperand(1)};
   return DAG.getMemIntrinsicNode(SystemZISD::PREFETCH, DL,
                                  Node->getVTList(), Ops,
                                  Node->getMemoryVT(), Node->getMemOperand());
 }
 
 // Convert condition code in CCReg to an i32 value.
 static SDValue getCCResult(SelectionDAG &DAG, SDValue CCReg) {
   SDLoc DL(CCReg);
   SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
   return DAG.getNode(ISD::SRL, DL, MVT::i32, IPM,
                      DAG.getConstant(SystemZ::IPM_CC, DL, MVT::i32));
 }
 
 SDValue
 SystemZTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
                                               SelectionDAG &DAG) const {
   unsigned Opcode, CCValid;
   if (isIntrinsicWithCCAndChain(Op, Opcode, CCValid)) {
     assert(Op->getNumValues() == 2 && "Expected only CC result and chain");
     SDNode *Node = emitIntrinsicWithCCAndChain(DAG, Op, Opcode);
     SDValue CC = getCCResult(DAG, SDValue(Node, 0));
     DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), CC);
     return SDValue();
   }
 
   return SDValue();
 }
 
 SDValue
 SystemZTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
                                                SelectionDAG &DAG) const {
   unsigned Opcode, CCValid;
   if (isIntrinsicWithCC(Op, Opcode, CCValid)) {
     SDNode *Node = emitIntrinsicWithCC(DAG, Op, Opcode);
     if (Op->getNumValues() == 1)
       return getCCResult(DAG, SDValue(Node, 0));
     assert(Op->getNumValues() == 2 && "Expected a CC and non-CC result");
     return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op), Op->getVTList(),
                        SDValue(Node, 0), getCCResult(DAG, SDValue(Node, 1)));
   }
 
   unsigned Id = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   switch (Id) {
   case Intrinsic::thread_pointer:
     return lowerThreadPointer(SDLoc(Op), DAG);
 
   case Intrinsic::s390_vpdi:
     return DAG.getNode(SystemZISD::PERMUTE_DWORDS, SDLoc(Op), Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
 
   case Intrinsic::s390_vperm:
     return DAG.getNode(SystemZISD::PERMUTE, SDLoc(Op), Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
 
   case Intrinsic::s390_vuphb:
   case Intrinsic::s390_vuphh:
   case Intrinsic::s390_vuphf:
     return DAG.getNode(SystemZISD::UNPACK_HIGH, SDLoc(Op), Op.getValueType(),
                        Op.getOperand(1));
 
   case Intrinsic::s390_vuplhb:
   case Intrinsic::s390_vuplhh:
   case Intrinsic::s390_vuplhf:
     return DAG.getNode(SystemZISD::UNPACKL_HIGH, SDLoc(Op), Op.getValueType(),
                        Op.getOperand(1));
 
   case Intrinsic::s390_vuplb:
   case Intrinsic::s390_vuplhw:
   case Intrinsic::s390_vuplf:
     return DAG.getNode(SystemZISD::UNPACK_LOW, SDLoc(Op), Op.getValueType(),
                        Op.getOperand(1));
 
   case Intrinsic::s390_vupllb:
   case Intrinsic::s390_vupllh:
   case Intrinsic::s390_vupllf:
     return DAG.getNode(SystemZISD::UNPACKL_LOW, SDLoc(Op), Op.getValueType(),
                        Op.getOperand(1));
 
   case Intrinsic::s390_vsumb:
   case Intrinsic::s390_vsumh:
   case Intrinsic::s390_vsumgh:
   case Intrinsic::s390_vsumgf:
   case Intrinsic::s390_vsumqf:
   case Intrinsic::s390_vsumqg:
     return DAG.getNode(SystemZISD::VSUM, SDLoc(Op), Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
   }
 
   return SDValue();
 }
 
 namespace {
 // Says that SystemZISD operation Opcode can be used to perform the equivalent
 // of a VPERM with permute vector Bytes.  If Opcode takes three operands,
 // Operand is the constant third operand, otherwise it is the number of
 // bytes in each element of the result.
 struct Permute {
   unsigned Opcode;
   unsigned Operand;
   unsigned char Bytes[SystemZ::VectorBytes];
 };
 }
 
 static const Permute PermuteForms[] = {
   // VMRHG
   { SystemZISD::MERGE_HIGH, 8,
     { 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 } },
   // VMRHF
   { SystemZISD::MERGE_HIGH, 4,
     { 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 } },
   // VMRHH
   { SystemZISD::MERGE_HIGH, 2,
     { 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23 } },
   // VMRHB
   { SystemZISD::MERGE_HIGH, 1,
     { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23 } },
   // VMRLG
   { SystemZISD::MERGE_LOW, 8,
     { 8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31 } },
   // VMRLF
   { SystemZISD::MERGE_LOW, 4,
     { 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 } },
   // VMRLH
   { SystemZISD::MERGE_LOW, 2,
     { 8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31 } },
   // VMRLB
   { SystemZISD::MERGE_LOW, 1,
     { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31 } },
   // VPKG
   { SystemZISD::PACK, 4,
     { 4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31 } },
   // VPKF
   { SystemZISD::PACK, 2,
     { 2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31 } },
   // VPKH
   { SystemZISD::PACK, 1,
     { 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 } },
   // VPDI V1, V2, 4  (low half of V1, high half of V2)
   { SystemZISD::PERMUTE_DWORDS, 4,
     { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 } },
   // VPDI V1, V2, 1  (high half of V1, low half of V2)
   { SystemZISD::PERMUTE_DWORDS, 1,
     { 0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31 } }
 };
 
 // Called after matching a vector shuffle against a particular pattern.
 // Both the original shuffle and the pattern have two vector operands.
 // OpNos[0] is the operand of the original shuffle that should be used for
 // operand 0 of the pattern, or -1 if operand 0 of the pattern can be anything.
 // OpNos[1] is the same for operand 1 of the pattern.  Resolve these -1s and
 // set OpNo0 and OpNo1 to the shuffle operands that should actually be used
 // for operands 0 and 1 of the pattern.
 static bool chooseShuffleOpNos(int *OpNos, unsigned &OpNo0, unsigned &OpNo1) {
   if (OpNos[0] < 0) {
     if (OpNos[1] < 0)
       return false;
     OpNo0 = OpNo1 = OpNos[1];
   } else if (OpNos[1] < 0) {
     OpNo0 = OpNo1 = OpNos[0];
   } else {
     OpNo0 = OpNos[0];
     OpNo1 = OpNos[1];
   }
   return true;
 }
 
 // Bytes is a VPERM-like permute vector, except that -1 is used for
 // undefined bytes.  Return true if the VPERM can be implemented using P.
 // When returning true set OpNo0 to the VPERM operand that should be
 // used for operand 0 of P and likewise OpNo1 for operand 1 of P.
 //
 // For example, if swapping the VPERM operands allows P to match, OpNo0
 // will be 1 and OpNo1 will be 0.  If instead Bytes only refers to one
 // operand, but rewriting it to use two duplicated operands allows it to
 // match P, then OpNo0 and OpNo1 will be the same.
 static bool matchPermute(const SmallVectorImpl<int> &Bytes, const Permute &P,
                          unsigned &OpNo0, unsigned &OpNo1) {
   int OpNos[] = { -1, -1 };
   for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
     int Elt = Bytes[I];
     if (Elt >= 0) {
       // Make sure that the two permute vectors use the same suboperand
       // byte number.  Only the operand numbers (the high bits) are
       // allowed to differ.
       if ((Elt ^ P.Bytes[I]) & (SystemZ::VectorBytes - 1))
         return false;
       int ModelOpNo = P.Bytes[I] / SystemZ::VectorBytes;
       int RealOpNo = unsigned(Elt) / SystemZ::VectorBytes;
       // Make sure that the operand mappings are consistent with previous
       // elements.
       if (OpNos[ModelOpNo] == 1 - RealOpNo)
         return false;
       OpNos[ModelOpNo] = RealOpNo;
     }
   }
   return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
 }
 
 // As above, but search for a matching permute.
 static const Permute *matchPermute(const SmallVectorImpl<int> &Bytes,
                                    unsigned &OpNo0, unsigned &OpNo1) {
   for (auto &P : PermuteForms)
     if (matchPermute(Bytes, P, OpNo0, OpNo1))
       return &P;
   return nullptr;
 }
 
 // Bytes is a VPERM-like permute vector, except that -1 is used for
 // undefined bytes.  This permute is an operand of an outer permute.
 // See whether redistributing the -1 bytes gives a shuffle that can be
 // implemented using P.  If so, set Transform to a VPERM-like permute vector
 // that, when applied to the result of P, gives the original permute in Bytes.
 static bool matchDoublePermute(const SmallVectorImpl<int> &Bytes,
                                const Permute &P,
                                SmallVectorImpl<int> &Transform) {
   unsigned To = 0;
   for (unsigned From = 0; From < SystemZ::VectorBytes; ++From) {
     int Elt = Bytes[From];
     if (Elt < 0)
       // Byte number From of the result is undefined.
       Transform[From] = -1;
     else {
       while (P.Bytes[To] != Elt) {
         To += 1;
         if (To == SystemZ::VectorBytes)
           return false;
       }
       Transform[From] = To;
     }
   }
   return true;
 }
 
 // As above, but search for a matching permute.
 static const Permute *matchDoublePermute(const SmallVectorImpl<int> &Bytes,
                                          SmallVectorImpl<int> &Transform) {
   for (auto &P : PermuteForms)
     if (matchDoublePermute(Bytes, P, Transform))
       return &P;
   return nullptr;
 }
 
 // Convert the mask of the given shuffle op into a byte-level mask,
 // as if it had type vNi8.
 static bool getVPermMask(SDValue ShuffleOp,
                          SmallVectorImpl<int> &Bytes) {
   EVT VT = ShuffleOp.getValueType();
   unsigned NumElements = VT.getVectorNumElements();
   unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
 
   if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(ShuffleOp)) {
     Bytes.resize(NumElements * BytesPerElement, -1);
     for (unsigned I = 0; I < NumElements; ++I) {
       int Index = VSN->getMaskElt(I);
       if (Index >= 0)
         for (unsigned J = 0; J < BytesPerElement; ++J)
           Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
     }
     return true;
   }
   if (SystemZISD::SPLAT == ShuffleOp.getOpcode() &&
       isa<ConstantSDNode>(ShuffleOp.getOperand(1))) {
     unsigned Index = ShuffleOp.getConstantOperandVal(1);
     Bytes.resize(NumElements * BytesPerElement, -1);
     for (unsigned I = 0; I < NumElements; ++I)
       for (unsigned J = 0; J < BytesPerElement; ++J)
         Bytes[I * BytesPerElement + J] = Index * BytesPerElement + J;
     return true;
   }
   return false;
 }
 
 // Bytes is a VPERM-like permute vector, except that -1 is used for
 // undefined bytes.  See whether bytes [Start, Start + BytesPerElement) of
 // the result come from a contiguous sequence of bytes from one input.
 // Set Base to the selector for the first byte if so.
 static bool getShuffleInput(const SmallVectorImpl<int> &Bytes, unsigned Start,
                             unsigned BytesPerElement, int &Base) {
   Base = -1;
   for (unsigned I = 0; I < BytesPerElement; ++I) {
     if (Bytes[Start + I] >= 0) {
       unsigned Elem = Bytes[Start + I];
       if (Base < 0) {
         Base = Elem - I;
         // Make sure the bytes would come from one input operand.
         if (unsigned(Base) % Bytes.size() + BytesPerElement > Bytes.size())
           return false;
       } else if (unsigned(Base) != Elem - I)
         return false;
     }
   }
   return true;
 }
 
 // Bytes is a VPERM-like permute vector, except that -1 is used for
 // undefined bytes.  Return true if it can be performed using VSLDB.
 // When returning true, set StartIndex to the shift amount and OpNo0
 // and OpNo1 to the VPERM operands that should be used as the first
 // and second shift operand respectively.
 static bool isShlDoublePermute(const SmallVectorImpl<int> &Bytes,
                                unsigned &StartIndex, unsigned &OpNo0,
                                unsigned &OpNo1) {
   int OpNos[] = { -1, -1 };
   int Shift = -1;
   for (unsigned I = 0; I < 16; ++I) {
     int Index = Bytes[I];
     if (Index >= 0) {
       int ExpectedShift = (Index - I) % SystemZ::VectorBytes;
       int ModelOpNo = unsigned(ExpectedShift + I) / SystemZ::VectorBytes;
       int RealOpNo = unsigned(Index) / SystemZ::VectorBytes;
       if (Shift < 0)
         Shift = ExpectedShift;
       else if (Shift != ExpectedShift)
         return false;
       // Make sure that the operand mappings are consistent with previous
       // elements.
       if (OpNos[ModelOpNo] == 1 - RealOpNo)
         return false;
       OpNos[ModelOpNo] = RealOpNo;
     }
   }
   StartIndex = Shift;
   return chooseShuffleOpNos(OpNos, OpNo0, OpNo1);
 }
 
 // Create a node that performs P on operands Op0 and Op1, casting the
 // operands to the appropriate type.  The type of the result is determined by P.
 static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
                               const Permute &P, SDValue Op0, SDValue Op1) {
   // VPDI (PERMUTE_DWORDS) always operates on v2i64s.  The input
   // elements of a PACK are twice as wide as the outputs.
   unsigned InBytes = (P.Opcode == SystemZISD::PERMUTE_DWORDS ? 8 :
                       P.Opcode == SystemZISD::PACK ? P.Operand * 2 :
                       P.Operand);
   // Cast both operands to the appropriate type.
   MVT InVT = MVT::getVectorVT(MVT::getIntegerVT(InBytes * 8),
                               SystemZ::VectorBytes / InBytes);
   Op0 = DAG.getNode(ISD::BITCAST, DL, InVT, Op0);
   Op1 = DAG.getNode(ISD::BITCAST, DL, InVT, Op1);
   SDValue Op;
   if (P.Opcode == SystemZISD::PERMUTE_DWORDS) {
     SDValue Op2 = DAG.getTargetConstant(P.Operand, DL, MVT::i32);
     Op = DAG.getNode(SystemZISD::PERMUTE_DWORDS, DL, InVT, Op0, Op1, Op2);
   } else if (P.Opcode == SystemZISD::PACK) {
     MVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(P.Operand * 8),
                                  SystemZ::VectorBytes / P.Operand);
     Op = DAG.getNode(SystemZISD::PACK, DL, OutVT, Op0, Op1);
   } else {
     Op = DAG.getNode(P.Opcode, DL, InVT, Op0, Op1);
   }
   return Op;
 }
 
 static bool isZeroVector(SDValue N) {
   if (N->getOpcode() == ISD::BITCAST)
     N = N->getOperand(0);
   if (N->getOpcode() == ISD::SPLAT_VECTOR)
     if (auto *Op = dyn_cast<ConstantSDNode>(N->getOperand(0)))
       return Op->getZExtValue() == 0;
   return ISD::isBuildVectorAllZeros(N.getNode());
 }
 
 // Return the index of the zero/undef vector, or UINT32_MAX if not found.
 static uint32_t findZeroVectorIdx(SDValue *Ops, unsigned Num) {
   for (unsigned I = 0; I < Num ; I++)
     if (isZeroVector(Ops[I]))
       return I;
   return UINT32_MAX;
 }
 
 // Bytes is a VPERM-like permute vector, except that -1 is used for
 // undefined bytes.  Implement it on operands Ops[0] and Ops[1] using
 // VSLDB or VPERM.
 static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
                                      SDValue *Ops,
                                      const SmallVectorImpl<int> &Bytes) {
   for (unsigned I = 0; I < 2; ++I)
     Ops[I] = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, Ops[I]);
 
   // First see whether VSLDB can be used.
   unsigned StartIndex, OpNo0, OpNo1;
   if (isShlDoublePermute(Bytes, StartIndex, OpNo0, OpNo1))
     return DAG.getNode(SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, Ops[OpNo0],
                        Ops[OpNo1],
                        DAG.getTargetConstant(StartIndex, DL, MVT::i32));
 
   // Fall back on VPERM.  Construct an SDNode for the permute vector.  Try to
   // eliminate a zero vector by reusing any zero index in the permute vector.
   unsigned ZeroVecIdx = findZeroVectorIdx(&Ops[0], 2);
   if (ZeroVecIdx != UINT32_MAX) {
     bool MaskFirst = true;
     int ZeroIdx = -1;
     for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
       unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
       unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
       if (OpNo == ZeroVecIdx && I == 0) {
         // If the first byte is zero, use mask as first operand.
         ZeroIdx = 0;
         break;
       }
       if (OpNo != ZeroVecIdx && Byte == 0) {
         // If mask contains a zero, use it by placing that vector first.
         ZeroIdx = I + SystemZ::VectorBytes;
         MaskFirst = false;
         break;
       }
     }
     if (ZeroIdx != -1) {
       SDValue IndexNodes[SystemZ::VectorBytes];
       for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
         if (Bytes[I] >= 0) {
           unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
           unsigned Byte = unsigned(Bytes[I]) % SystemZ::VectorBytes;
           if (OpNo == ZeroVecIdx)
             IndexNodes[I] = DAG.getConstant(ZeroIdx, DL, MVT::i32);
           else {
             unsigned BIdx = MaskFirst ? Byte + SystemZ::VectorBytes : Byte;
             IndexNodes[I] = DAG.getConstant(BIdx, DL, MVT::i32);
           }
         } else
           IndexNodes[I] = DAG.getUNDEF(MVT::i32);
       }
       SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, IndexNodes);
       SDValue Src = ZeroVecIdx == 0 ? Ops[1] : Ops[0];
       if (MaskFirst)
         return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Mask, Src,
                            Mask);
       else
         return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Src, Mask,
                            Mask);
     }
   }
 
   SDValue IndexNodes[SystemZ::VectorBytes];
   for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
     if (Bytes[I] >= 0)
       IndexNodes[I] = DAG.getConstant(Bytes[I], DL, MVT::i32);
     else
       IndexNodes[I] = DAG.getUNDEF(MVT::i32);
   SDValue Op2 = DAG.getBuildVector(MVT::v16i8, DL, IndexNodes);
   return DAG.getNode(SystemZISD::PERMUTE, DL, MVT::v16i8, Ops[0],
                      (!Ops[1].isUndef() ? Ops[1] : Ops[0]), Op2);
 }
 
 namespace {
 // Describes a general N-operand vector shuffle.
 struct GeneralShuffle {
   GeneralShuffle(EVT vt) : VT(vt), UnpackFromEltSize(UINT_MAX) {}
   void addUndef();
   bool add(SDValue, unsigned);
   SDValue getNode(SelectionDAG &, const SDLoc &);
   void tryPrepareForUnpack();
   bool unpackWasPrepared() { return UnpackFromEltSize <= 4; }
   SDValue insertUnpackIfPrepared(SelectionDAG &DAG, const SDLoc &DL, SDValue Op);
 
   // The operands of the shuffle.
   SmallVector<SDValue, SystemZ::VectorBytes> Ops;
 
   // Index I is -1 if byte I of the result is undefined.  Otherwise the
   // result comes from byte Bytes[I] % SystemZ::VectorBytes of operand
   // Bytes[I] / SystemZ::VectorBytes.
   SmallVector<int, SystemZ::VectorBytes> Bytes;
 
   // The type of the shuffle result.
   EVT VT;
 
   // Holds a value of 1, 2 or 4 if a final unpack has been prepared for.
   unsigned UnpackFromEltSize;
 };
 }
 
 // Add an extra undefined element to the shuffle.
 void GeneralShuffle::addUndef() {
   unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
   for (unsigned I = 0; I < BytesPerElement; ++I)
     Bytes.push_back(-1);
 }
 
 // Add an extra element to the shuffle, taking it from element Elem of Op.
 // A null Op indicates a vector input whose value will be calculated later;
 // there is at most one such input per shuffle and it always has the same
 // type as the result. Aborts and returns false if the source vector elements
 // of an EXTRACT_VECTOR_ELT are smaller than the destination elements. Per
 // LLVM they become implicitly extended, but this is rare and not optimized.
 bool GeneralShuffle::add(SDValue Op, unsigned Elem) {
   unsigned BytesPerElement = VT.getVectorElementType().getStoreSize();
 
   // The source vector can have wider elements than the result,
   // either through an explicit TRUNCATE or because of type legalization.
   // We want the least significant part.
   EVT FromVT = Op.getNode() ? Op.getValueType() : VT;
   unsigned FromBytesPerElement = FromVT.getVectorElementType().getStoreSize();
 
   // Return false if the source elements are smaller than their destination
   // elements.
   if (FromBytesPerElement < BytesPerElement)
     return false;
 
   unsigned Byte = ((Elem * FromBytesPerElement) % SystemZ::VectorBytes +
                    (FromBytesPerElement - BytesPerElement));
 
   // Look through things like shuffles and bitcasts.
   while (Op.getNode()) {
     if (Op.getOpcode() == ISD::BITCAST)
       Op = Op.getOperand(0);
     else if (Op.getOpcode() == ISD::VECTOR_SHUFFLE && Op.hasOneUse()) {
       // See whether the bytes we need come from a contiguous part of one
       // operand.
       SmallVector<int, SystemZ::VectorBytes> OpBytes;
       if (!getVPermMask(Op, OpBytes))
         break;
       int NewByte;
       if (!getShuffleInput(OpBytes, Byte, BytesPerElement, NewByte))
         break;
       if (NewByte < 0) {
         addUndef();
         return true;
       }
       Op = Op.getOperand(unsigned(NewByte) / SystemZ::VectorBytes);
       Byte = unsigned(NewByte) % SystemZ::VectorBytes;
     } else if (Op.isUndef()) {
       addUndef();
       return true;
     } else
       break;
   }
 
   // Make sure that the source of the extraction is in Ops.
   unsigned OpNo = 0;
   for (; OpNo < Ops.size(); ++OpNo)
     if (Ops[OpNo] == Op)
       break;
   if (OpNo == Ops.size())
     Ops.push_back(Op);
 
   // Add the element to Bytes.
   unsigned Base = OpNo * SystemZ::VectorBytes + Byte;
   for (unsigned I = 0; I < BytesPerElement; ++I)
     Bytes.push_back(Base + I);
 
   return true;
 }
 
 // Return SDNodes for the completed shuffle.
 SDValue GeneralShuffle::getNode(SelectionDAG &DAG, const SDLoc &DL) {
   assert(Bytes.size() == SystemZ::VectorBytes && "Incomplete vector");
 
   if (Ops.size() == 0)
     return DAG.getUNDEF(VT);
 
   // Use a single unpack if possible as the last operation.
   tryPrepareForUnpack();
 
   // Make sure that there are at least two shuffle operands.
   if (Ops.size() == 1)
     Ops.push_back(DAG.getUNDEF(MVT::v16i8));
 
   // Create a tree of shuffles, deferring root node until after the loop.
   // Try to redistribute the undefined elements of non-root nodes so that
   // the non-root shuffles match something like a pack or merge, then adjust
   // the parent node's permute vector to compensate for the new order.
   // Among other things, this copes with vectors like <2 x i16> that were
   // padded with undefined elements during type legalization.
   //
   // In the best case this redistribution will lead to the whole tree
   // using packs and merges.  It should rarely be a loss in other cases.
   unsigned Stride = 1;
   for (; Stride * 2 < Ops.size(); Stride *= 2) {
     for (unsigned I = 0; I < Ops.size() - Stride; I += Stride * 2) {
       SDValue SubOps[] = { Ops[I], Ops[I + Stride] };
 
       // Create a mask for just these two operands.
       SmallVector<int, SystemZ::VectorBytes> NewBytes(SystemZ::VectorBytes);
       for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
         unsigned OpNo = unsigned(Bytes[J]) / SystemZ::VectorBytes;
         unsigned Byte = unsigned(Bytes[J]) % SystemZ::VectorBytes;
         if (OpNo == I)
           NewBytes[J] = Byte;
         else if (OpNo == I + Stride)
           NewBytes[J] = SystemZ::VectorBytes + Byte;
         else
           NewBytes[J] = -1;
       }
       // See if it would be better to reorganize NewMask to avoid using VPERM.
       SmallVector<int, SystemZ::VectorBytes> NewBytesMap(SystemZ::VectorBytes);
       if (const Permute *P = matchDoublePermute(NewBytes, NewBytesMap)) {
         Ops[I] = getPermuteNode(DAG, DL, *P, SubOps[0], SubOps[1]);
         // Applying NewBytesMap to Ops[I] gets back to NewBytes.
         for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
           if (NewBytes[J] >= 0) {
             assert(unsigned(NewBytesMap[J]) < SystemZ::VectorBytes &&
                    "Invalid double permute");
             Bytes[J] = I * SystemZ::VectorBytes + NewBytesMap[J];
           } else
             assert(NewBytesMap[J] < 0 && "Invalid double permute");
         }
       } else {
         // Just use NewBytes on the operands.
         Ops[I] = getGeneralPermuteNode(DAG, DL, SubOps, NewBytes);
         for (unsigned J = 0; J < SystemZ::VectorBytes; ++J)
           if (NewBytes[J] >= 0)
             Bytes[J] = I * SystemZ::VectorBytes + J;
       }
     }
   }
 
   // Now we just have 2 inputs.  Put the second operand in Ops[1].
   if (Stride > 1) {
     Ops[1] = Ops[Stride];
     for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
       if (Bytes[I] >= int(SystemZ::VectorBytes))
         Bytes[I] -= (Stride - 1) * SystemZ::VectorBytes;
   }
 
   // Look for an instruction that can do the permute without resorting
   // to VPERM.
   unsigned OpNo0, OpNo1;
   SDValue Op;
   if (unpackWasPrepared() && Ops[1].isUndef())
     Op = Ops[0];
   else if (const Permute *P = matchPermute(Bytes, OpNo0, OpNo1))
     Op = getPermuteNode(DAG, DL, *P, Ops[OpNo0], Ops[OpNo1]);
   else
     Op = getGeneralPermuteNode(DAG, DL, &Ops[0], Bytes);
 
   Op = insertUnpackIfPrepared(DAG, DL, Op);
 
   return DAG.getNode(ISD::BITCAST, DL, VT, Op);
 }
 
 #ifndef NDEBUG
 static void dumpBytes(const SmallVectorImpl<int> &Bytes, std::string Msg) {
   dbgs() << Msg.c_str() << " { ";
   for (unsigned i = 0; i < Bytes.size(); i++)
     dbgs() << Bytes[i] << " ";
   dbgs() << "}\n";
 }
 #endif
 
 // If the Bytes vector matches an unpack operation, prepare to do the unpack
 // after all else by removing the zero vector and the effect of the unpack on
 // Bytes.
 void GeneralShuffle::tryPrepareForUnpack() {
   uint32_t ZeroVecOpNo = findZeroVectorIdx(&Ops[0], Ops.size());
   if (ZeroVecOpNo == UINT32_MAX || Ops.size() == 1)
     return;
 
   // Only do this if removing the zero vector reduces the depth, otherwise
   // the critical path will increase with the final unpack.
   if (Ops.size() > 2 &&
       Log2_32_Ceil(Ops.size()) == Log2_32_Ceil(Ops.size() - 1))
     return;
 
   // Find an unpack that would allow removing the zero vector from Ops.
   UnpackFromEltSize = 1;
   for (; UnpackFromEltSize <= 4; UnpackFromEltSize *= 2) {
     bool MatchUnpack = true;
     SmallVector<int, SystemZ::VectorBytes> SrcBytes;
     for (unsigned Elt = 0; Elt < SystemZ::VectorBytes; Elt++) {
       unsigned ToEltSize = UnpackFromEltSize * 2;
       bool IsZextByte = (Elt % ToEltSize) < UnpackFromEltSize;
       if (!IsZextByte)
         SrcBytes.push_back(Bytes[Elt]);
       if (Bytes[Elt] != -1) {
         unsigned OpNo = unsigned(Bytes[Elt]) / SystemZ::VectorBytes;
         if (IsZextByte != (OpNo == ZeroVecOpNo)) {
           MatchUnpack = false;
           break;
         }
       }
     }
     if (MatchUnpack) {
       if (Ops.size() == 2) {
         // Don't use unpack if a single source operand needs rearrangement.
         for (unsigned i = 0; i < SystemZ::VectorBytes / 2; i++)
           if (SrcBytes[i] != -1 && SrcBytes[i] % 16 != int(i)) {
             UnpackFromEltSize = UINT_MAX;
             return;
           }
       }
       break;
     }
   }
   if (UnpackFromEltSize > 4)
     return;
 
   LLVM_DEBUG(dbgs() << "Preparing for final unpack of element size "
              << UnpackFromEltSize << ". Zero vector is Op#" << ZeroVecOpNo
              << ".\n";
              dumpBytes(Bytes, "Original Bytes vector:"););
 
   // Apply the unpack in reverse to the Bytes array.
   unsigned B = 0;
   for (unsigned Elt = 0; Elt < SystemZ::VectorBytes;) {
     Elt += UnpackFromEltSize;
     for (unsigned i = 0; i < UnpackFromEltSize; i++, Elt++, B++)
       Bytes[B] = Bytes[Elt];
   }
   while (B < SystemZ::VectorBytes)
     Bytes[B++] = -1;
 
   // Remove the zero vector from Ops
   Ops.erase(&Ops[ZeroVecOpNo]);
   for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
     if (Bytes[I] >= 0) {
       unsigned OpNo = unsigned(Bytes[I]) / SystemZ::VectorBytes;
       if (OpNo > ZeroVecOpNo)
         Bytes[I] -= SystemZ::VectorBytes;
     }
 
   LLVM_DEBUG(dumpBytes(Bytes, "Resulting Bytes vector, zero vector removed:");
              dbgs() << "\n";);
 }
 
 SDValue GeneralShuffle::insertUnpackIfPrepared(SelectionDAG &DAG,
                                                const SDLoc &DL,
                                                SDValue Op) {
   if (!unpackWasPrepared())
     return Op;
   unsigned InBits = UnpackFromEltSize * 8;
   EVT InVT = MVT::getVectorVT(MVT::getIntegerVT(InBits),
                                 SystemZ::VectorBits / InBits);
   SDValue PackedOp = DAG.getNode(ISD::BITCAST, DL, InVT, Op);
   unsigned OutBits = InBits * 2;
   EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(OutBits),
                                SystemZ::VectorBits / OutBits);
   return DAG.getNode(SystemZISD::UNPACKL_HIGH, DL, OutVT, PackedOp);
 }
 
 // Return true if the given BUILD_VECTOR is a scalar-to-vector conversion.
 static bool isScalarToVector(SDValue Op) {
   for (unsigned I = 1, E = Op.getNumOperands(); I != E; ++I)
     if (!Op.getOperand(I).isUndef())
       return false;
   return true;
 }
 
 // Return a vector of type VT that contains Value in the first element.
 // The other elements don't matter.
 static SDValue buildScalarToVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
                                    SDValue Value) {
   // If we have a constant, replicate it to all elements and let the
   // BUILD_VECTOR lowering take care of it.
   if (Value.getOpcode() == ISD::Constant ||
       Value.getOpcode() == ISD::ConstantFP) {
     SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Value);
     return DAG.getBuildVector(VT, DL, Ops);
   }
   if (Value.isUndef())
     return DAG.getUNDEF(VT);
   return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Value);
 }
 
 // Return a vector of type VT in which Op0 is in element 0 and Op1 is in
 // element 1.  Used for cases in which replication is cheap.
 static SDValue buildMergeScalars(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
                                  SDValue Op0, SDValue Op1) {
   if (Op0.isUndef()) {
     if (Op1.isUndef())
       return DAG.getUNDEF(VT);
     return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op1);
   }
   if (Op1.isUndef())
     return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0);
   return DAG.getNode(SystemZISD::MERGE_HIGH, DL, VT,
                      buildScalarToVector(DAG, DL, VT, Op0),
                      buildScalarToVector(DAG, DL, VT, Op1));
 }
 
 // Extend GPR scalars Op0 and Op1 to doublewords and return a v2i64
 // vector for them.
 static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0,
                           SDValue Op1) {
   if (Op0.isUndef() && Op1.isUndef())
     return DAG.getUNDEF(MVT::v2i64);
   // If one of the two inputs is undefined then replicate the other one,
   // in order to avoid using another register unnecessarily.
   if (Op0.isUndef())
     Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
   else if (Op1.isUndef())
     Op0 = Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
   else {
     Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
     Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op1);
   }
   return DAG.getNode(SystemZISD::JOIN_DWORDS, DL, MVT::v2i64, Op0, Op1);
 }
 
 // If a BUILD_VECTOR contains some EXTRACT_VECTOR_ELTs, it's usually
 // better to use VECTOR_SHUFFLEs on them, only using BUILD_VECTOR for
 // the non-EXTRACT_VECTOR_ELT elements.  See if the given BUILD_VECTOR
 // would benefit from this representation and return it if so.
 static SDValue tryBuildVectorShuffle(SelectionDAG &DAG,
                                      BuildVectorSDNode *BVN) {
   EVT VT = BVN->getValueType(0);
   unsigned NumElements = VT.getVectorNumElements();
 
   // Represent the BUILD_VECTOR as an N-operand VECTOR_SHUFFLE-like operation
   // on byte vectors.  If there are non-EXTRACT_VECTOR_ELT elements that still
   // need a BUILD_VECTOR, add an additional placeholder operand for that
   // BUILD_VECTOR and store its operands in ResidueOps.
   GeneralShuffle GS(VT);
   SmallVector<SDValue, SystemZ::VectorBytes> ResidueOps;
   bool FoundOne = false;
   for (unsigned I = 0; I < NumElements; ++I) {
     SDValue Op = BVN->getOperand(I);
     if (Op.getOpcode() == ISD::TRUNCATE)
       Op = Op.getOperand(0);
     if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
         Op.getOperand(1).getOpcode() == ISD::Constant) {
       unsigned Elem = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
       if (!GS.add(Op.getOperand(0), Elem))
         return SDValue();
       FoundOne = true;
     } else if (Op.isUndef()) {
       GS.addUndef();
     } else {
       if (!GS.add(SDValue(), ResidueOps.size()))
         return SDValue();
       ResidueOps.push_back(BVN->getOperand(I));
     }
   }
 
   // Nothing to do if there are no EXTRACT_VECTOR_ELTs.
   if (!FoundOne)
     return SDValue();
 
   // Create the BUILD_VECTOR for the remaining elements, if any.
   if (!ResidueOps.empty()) {
     while (ResidueOps.size() < NumElements)
       ResidueOps.push_back(DAG.getUNDEF(ResidueOps[0].getValueType()));
     for (auto &Op : GS.Ops) {
       if (!Op.getNode()) {
         Op = DAG.getBuildVector(VT, SDLoc(BVN), ResidueOps);
         break;
       }
     }
   }
   return GS.getNode(DAG, SDLoc(BVN));
 }
 
 bool SystemZTargetLowering::isVectorElementLoad(SDValue Op) const {
   if (Op.getOpcode() == ISD::LOAD && cast<LoadSDNode>(Op)->isUnindexed())
     return true;
   if (Subtarget.hasVectorEnhancements2() && Op.getOpcode() == SystemZISD::LRV)
     return true;
   return false;
 }
 
 // Combine GPR scalar values Elems into a vector of type VT.
 SDValue
 SystemZTargetLowering::buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
                                    SmallVectorImpl<SDValue> &Elems) const {
   // See whether there is a single replicated value.
   SDValue Single;
   unsigned int NumElements = Elems.size();
   unsigned int Count = 0;
   for (auto Elem : Elems) {
     if (!Elem.isUndef()) {
       if (!Single.getNode())
         Single = Elem;
       else if (Elem != Single) {
         Single = SDValue();
         break;
       }
       Count += 1;
     }
   }
   // There are three cases here:
   //
   // - if the only defined element is a loaded one, the best sequence
   //   is a replicating load.
   //
   // - otherwise, if the only defined element is an i64 value, we will
   //   end up with the same VLVGP sequence regardless of whether we short-cut
   //   for replication or fall through to the later code.
   //
   // - otherwise, if the only defined element is an i32 or smaller value,
   //   we would need 2 instructions to replicate it: VLVGP followed by VREPx.
   //   This is only a win if the single defined element is used more than once.
   //   In other cases we're better off using a single VLVGx.
   if (Single.getNode() && (Count > 1 || isVectorElementLoad(Single)))
     return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Single);
 
   // If all elements are loads, use VLREP/VLEs (below).
   bool AllLoads = true;
   for (auto Elem : Elems)
     if (!isVectorElementLoad(Elem)) {
       AllLoads = false;
       break;
     }
 
   // The best way of building a v2i64 from two i64s is to use VLVGP.
   if (VT == MVT::v2i64 && !AllLoads)
     return joinDwords(DAG, DL, Elems[0], Elems[1]);
 
   // Use a 64-bit merge high to combine two doubles.
   if (VT == MVT::v2f64 && !AllLoads)
     return buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
 
   // Build v4f32 values directly from the FPRs:
   //
   //   <Axxx> <Bxxx> <Cxxxx> <Dxxx>
   //         V              V         VMRHF
   //      <ABxx>         <CDxx>
   //                V                 VMRHG
   //              <ABCD>
   if (VT == MVT::v4f32 && !AllLoads) {
     SDValue Op01 = buildMergeScalars(DAG, DL, VT, Elems[0], Elems[1]);
     SDValue Op23 = buildMergeScalars(DAG, DL, VT, Elems[2], Elems[3]);
     // Avoid unnecessary undefs by reusing the other operand.
     if (Op01.isUndef())
       Op01 = Op23;
     else if (Op23.isUndef())
       Op23 = Op01;
     // Merging identical replications is a no-op.
     if (Op01.getOpcode() == SystemZISD::REPLICATE && Op01 == Op23)
       return Op01;
     Op01 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op01);
     Op23 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Op23);
     SDValue Op = DAG.getNode(SystemZISD::MERGE_HIGH,
                              DL, MVT::v2i64, Op01, Op23);
     return DAG.getNode(ISD::BITCAST, DL, VT, Op);
   }
 
   // Collect the constant terms.
   SmallVector<SDValue, SystemZ::VectorBytes> Constants(NumElements, SDValue());
   SmallVector<bool, SystemZ::VectorBytes> Done(NumElements, false);
 
   unsigned NumConstants = 0;
   for (unsigned I = 0; I < NumElements; ++I) {
     SDValue Elem = Elems[I];
     if (Elem.getOpcode() == ISD::Constant ||
         Elem.getOpcode() == ISD::ConstantFP) {
       NumConstants += 1;
       Constants[I] = Elem;
       Done[I] = true;
     }
   }
   // If there was at least one constant, fill in the other elements of
   // Constants with undefs to get a full vector constant and use that
   // as the starting point.
   SDValue Result;
   SDValue ReplicatedVal;
   if (NumConstants > 0) {
     for (unsigned I = 0; I < NumElements; ++I)
       if (!Constants[I].getNode())
         Constants[I] = DAG.getUNDEF(Elems[I].getValueType());
     Result = DAG.getBuildVector(VT, DL, Constants);
   } else {
     // Otherwise try to use VLREP or VLVGP to start the sequence in order to
     // avoid a false dependency on any previous contents of the vector
     // register.
 
     // Use a VLREP if at least one element is a load. Make sure to replicate
     // the load with the most elements having its value.
     std::map<const SDNode*, unsigned> UseCounts;
     SDNode *LoadMaxUses = nullptr;
     for (unsigned I = 0; I < NumElements; ++I)
       if (isVectorElementLoad(Elems[I])) {
         SDNode *Ld = Elems[I].getNode();
         UseCounts[Ld]++;
         if (LoadMaxUses == nullptr || UseCounts[LoadMaxUses] < UseCounts[Ld])
           LoadMaxUses = Ld;
       }
     if (LoadMaxUses != nullptr) {
       ReplicatedVal = SDValue(LoadMaxUses, 0);
       Result = DAG.getNode(SystemZISD::REPLICATE, DL, VT, ReplicatedVal);
     } else {
       // Try to use VLVGP.
       unsigned I1 = NumElements / 2 - 1;
       unsigned I2 = NumElements - 1;
       bool Def1 = !Elems[I1].isUndef();
       bool Def2 = !Elems[I2].isUndef();
       if (Def1 || Def2) {
         SDValue Elem1 = Elems[Def1 ? I1 : I2];
         SDValue Elem2 = Elems[Def2 ? I2 : I1];
         Result = DAG.getNode(ISD::BITCAST, DL, VT,
                              joinDwords(DAG, DL, Elem1, Elem2));
         Done[I1] = true;
         Done[I2] = true;
       } else
         Result = DAG.getUNDEF(VT);
     }
   }
 
   // Use VLVGx to insert the other elements.
   for (unsigned I = 0; I < NumElements; ++I)
     if (!Done[I] && !Elems[I].isUndef() && Elems[I] != ReplicatedVal)
       Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Result, Elems[I],
                            DAG.getConstant(I, DL, MVT::i32));
   return Result;
 }
 
 SDValue SystemZTargetLowering::lowerBUILD_VECTOR(SDValue Op,
                                                  SelectionDAG &DAG) const {
   auto *BVN = cast<BuildVectorSDNode>(Op.getNode());
   SDLoc DL(Op);
   EVT VT = Op.getValueType();
 
   if (BVN->isConstant()) {
     if (SystemZVectorConstantInfo(BVN).isVectorConstantLegal(Subtarget))
       return Op;
 
     // Fall back to loading it from memory.
     return SDValue();
   }
 
   // See if we should use shuffles to construct the vector from other vectors.
   if (SDValue Res = tryBuildVectorShuffle(DAG, BVN))
     return Res;
 
   // Detect SCALAR_TO_VECTOR conversions.
   if (isOperationLegal(ISD::SCALAR_TO_VECTOR, VT) && isScalarToVector(Op))
     return buildScalarToVector(DAG, DL, VT, Op.getOperand(0));
 
   // Otherwise use buildVector to build the vector up from GPRs.
   unsigned NumElements = Op.getNumOperands();
   SmallVector<SDValue, SystemZ::VectorBytes> Ops(NumElements);
   for (unsigned I = 0; I < NumElements; ++I)
     Ops[I] = Op.getOperand(I);
   return buildVector(DAG, DL, VT, Ops);
 }
 
 SDValue SystemZTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
                                                    SelectionDAG &DAG) const {
   auto *VSN = cast<ShuffleVectorSDNode>(Op.getNode());
   SDLoc DL(Op);
   EVT VT = Op.getValueType();
   unsigned NumElements = VT.getVectorNumElements();
 
   if (VSN->isSplat()) {
     SDValue Op0 = Op.getOperand(0);
     unsigned Index = VSN->getSplatIndex();
     assert(Index < VT.getVectorNumElements() &&
            "Splat index should be defined and in first operand");
     // See whether the value we're splatting is directly available as a scalar.
     if ((Index == 0 && Op0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
         Op0.getOpcode() == ISD::BUILD_VECTOR)
       return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0.getOperand(Index));
     // Otherwise keep it as a vector-to-vector operation.
     return DAG.getNode(SystemZISD::SPLAT, DL, VT, Op.getOperand(0),
                        DAG.getTargetConstant(Index, DL, MVT::i32));
   }
 
   GeneralShuffle GS(VT);
   for (unsigned I = 0; I < NumElements; ++I) {
     int Elt = VSN->getMaskElt(I);
     if (Elt < 0)
       GS.addUndef();
     else if (!GS.add(Op.getOperand(unsigned(Elt) / NumElements),
                      unsigned(Elt) % NumElements))
       return SDValue();
   }
   return GS.getNode(DAG, SDLoc(VSN));
 }
 
 SDValue SystemZTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
                                                      SelectionDAG &DAG) const {
   SDLoc DL(Op);
   // Just insert the scalar into element 0 of an undefined vector.
   return DAG.getNode(ISD::INSERT_VECTOR_ELT, DL,
                      Op.getValueType(), DAG.getUNDEF(Op.getValueType()),
                      Op.getOperand(0), DAG.getConstant(0, DL, MVT::i32));
 }
 
 SDValue SystemZTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
                                                       SelectionDAG &DAG) const {
   // Handle insertions of floating-point values.
   SDLoc DL(Op);
   SDValue Op0 = Op.getOperand(0);
   SDValue Op1 = Op.getOperand(1);
   SDValue Op2 = Op.getOperand(2);
   EVT VT = Op.getValueType();
 
   // Insertions into constant indices of a v2f64 can be done using VPDI.
   // However, if the inserted value is a bitcast or a constant then it's
   // better to use GPRs, as below.
   if (VT == MVT::v2f64 &&
       Op1.getOpcode() != ISD::BITCAST &&
       Op1.getOpcode() != ISD::ConstantFP &&
       Op2.getOpcode() == ISD::Constant) {
     uint64_t Index = cast<ConstantSDNode>(Op2)->getZExtValue();
     unsigned Mask = VT.getVectorNumElements() - 1;
     if (Index <= Mask)
       return Op;
   }
 
   // Otherwise bitcast to the equivalent integer form and insert via a GPR.
   MVT IntVT = MVT::getIntegerVT(VT.getScalarSizeInBits());
   MVT IntVecVT = MVT::getVectorVT(IntVT, VT.getVectorNumElements());
   SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntVecVT,
                             DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0),
                             DAG.getNode(ISD::BITCAST, DL, IntVT, Op1), Op2);
   return DAG.getNode(ISD::BITCAST, DL, VT, Res);
 }
 
 SDValue
 SystemZTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
                                                SelectionDAG &DAG) const {
   // Handle extractions of floating-point values.
   SDLoc DL(Op);
   SDValue Op0 = Op.getOperand(0);
   SDValue Op1 = Op.getOperand(1);
   EVT VT = Op.getValueType();
   EVT VecVT = Op0.getValueType();
 
   // Extractions of constant indices can be done directly.
   if (auto *CIndexN = dyn_cast<ConstantSDNode>(Op1)) {
     uint64_t Index = CIndexN->getZExtValue();
     unsigned Mask = VecVT.getVectorNumElements() - 1;
     if (Index <= Mask)
       return Op;
   }
 
   // Otherwise bitcast to the equivalent integer form and extract via a GPR.
   MVT IntVT = MVT::getIntegerVT(VT.getSizeInBits());
   MVT IntVecVT = MVT::getVectorVT(IntVT, VecVT.getVectorNumElements());
   SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntVT,
                             DAG.getNode(ISD::BITCAST, DL, IntVecVT, Op0), Op1);
   return DAG.getNode(ISD::BITCAST, DL, VT, Res);
 }
 
 SDValue SystemZTargetLowering::
 lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
   SDValue PackedOp = Op.getOperand(0);
   EVT OutVT = Op.getValueType();
   EVT InVT = PackedOp.getValueType();
   unsigned ToBits = OutVT.getScalarSizeInBits();
   unsigned FromBits = InVT.getScalarSizeInBits();
   do {
     FromBits *= 2;
     EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(FromBits),
                                  SystemZ::VectorBits / FromBits);
     PackedOp =
       DAG.getNode(SystemZISD::UNPACK_HIGH, SDLoc(PackedOp), OutVT, PackedOp);
   } while (FromBits != ToBits);
   return PackedOp;
 }
 
 // Lower a ZERO_EXTEND_VECTOR_INREG to a vector shuffle with a zero vector.
 SDValue SystemZTargetLowering::
 lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const {
   SDValue PackedOp = Op.getOperand(0);
   SDLoc DL(Op);
   EVT OutVT = Op.getValueType();
   EVT InVT = PackedOp.getValueType();
   unsigned InNumElts = InVT.getVectorNumElements();
   unsigned OutNumElts = OutVT.getVectorNumElements();
   unsigned NumInPerOut = InNumElts / OutNumElts;
 
   SDValue ZeroVec =
     DAG.getSplatVector(InVT, DL, DAG.getConstant(0, DL, InVT.getScalarType()));
 
   SmallVector<int, 16> Mask(InNumElts);
   unsigned ZeroVecElt = InNumElts;
   for (unsigned PackedElt = 0; PackedElt < OutNumElts; PackedElt++) {
     unsigned MaskElt = PackedElt * NumInPerOut;
     unsigned End = MaskElt + NumInPerOut - 1;
     for (; MaskElt < End; MaskElt++)
       Mask[MaskElt] = ZeroVecElt++;
     Mask[MaskElt] = PackedElt;
   }
   SDValue Shuf = DAG.getVectorShuffle(InVT, DL, PackedOp, ZeroVec, Mask);
   return DAG.getNode(ISD::BITCAST, DL, OutVT, Shuf);
 }
 
 SDValue SystemZTargetLowering::lowerShift(SDValue Op, SelectionDAG &DAG,
                                           unsigned ByScalar) const {
   // Look for cases where a vector shift can use the *_BY_SCALAR form.
   SDValue Op0 = Op.getOperand(0);
   SDValue Op1 = Op.getOperand(1);
   SDLoc DL(Op);
   EVT VT = Op.getValueType();
   unsigned ElemBitSize = VT.getScalarSizeInBits();
 
   // See whether the shift vector is a splat represented as BUILD_VECTOR.
   if (auto *BVN = dyn_cast<BuildVectorSDNode>(Op1)) {
     APInt SplatBits, SplatUndef;
     unsigned SplatBitSize;
     bool HasAnyUndefs;
     // Check for constant splats.  Use ElemBitSize as the minimum element
     // width and reject splats that need wider elements.
     if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
                              ElemBitSize, true) &&
         SplatBitSize == ElemBitSize) {
       SDValue Shift = DAG.getConstant(SplatBits.getZExtValue() & 0xfff,
                                       DL, MVT::i32);
       return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
     }
     // Check for variable splats.
     BitVector UndefElements;
     SDValue Splat = BVN->getSplatValue(&UndefElements);
     if (Splat) {
       // Since i32 is the smallest legal type, we either need a no-op
       // or a truncation.
       SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Splat);
       return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
     }
   }
 
   // See whether the shift vector is a splat represented as SHUFFLE_VECTOR,
   // and the shift amount is directly available in a GPR.
   if (auto *VSN = dyn_cast<ShuffleVectorSDNode>(Op1)) {
     if (VSN->isSplat()) {
       SDValue VSNOp0 = VSN->getOperand(0);
       unsigned Index = VSN->getSplatIndex();
       assert(Index < VT.getVectorNumElements() &&
              "Splat index should be defined and in first operand");
       if ((Index == 0 && VSNOp0.getOpcode() == ISD::SCALAR_TO_VECTOR) ||
           VSNOp0.getOpcode() == ISD::BUILD_VECTOR) {
         // Since i32 is the smallest legal type, we either need a no-op
         // or a truncation.
         SDValue Shift = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
                                     VSNOp0.getOperand(Index));
         return DAG.getNode(ByScalar, DL, VT, Op0, Shift);
       }
     }
   }
 
   // Otherwise just treat the current form as legal.
   return Op;
 }
 
 SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
                                               SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
   case ISD::FRAMEADDR:
     return lowerFRAMEADDR(Op, DAG);
   case ISD::RETURNADDR:
     return lowerRETURNADDR(Op, DAG);
   case ISD::BR_CC:
     return lowerBR_CC(Op, DAG);
   case ISD::SELECT_CC:
     return lowerSELECT_CC(Op, DAG);
   case ISD::SETCC:
     return lowerSETCC(Op, DAG);
   case ISD::STRICT_FSETCC:
     return lowerSTRICT_FSETCC(Op, DAG, false);
   case ISD::STRICT_FSETCCS:
     return lowerSTRICT_FSETCC(Op, DAG, true);
   case ISD::GlobalAddress:
     return lowerGlobalAddress(cast<GlobalAddressSDNode>(Op), DAG);
   case ISD::GlobalTLSAddress:
     return lowerGlobalTLSAddress(cast<GlobalAddressSDNode>(Op), DAG);
   case ISD::BlockAddress:
     return lowerBlockAddress(cast<BlockAddressSDNode>(Op), DAG);
   case ISD::JumpTable:
     return lowerJumpTable(cast<JumpTableSDNode>(Op), DAG);
   case ISD::ConstantPool:
     return lowerConstantPool(cast<ConstantPoolSDNode>(Op), DAG);
   case ISD::BITCAST:
     return lowerBITCAST(Op, DAG);
   case ISD::VASTART:
     return lowerVASTART(Op, DAG);
   case ISD::VACOPY:
     return lowerVACOPY(Op, DAG);
   case ISD::DYNAMIC_STACKALLOC:
     return lowerDYNAMIC_STACKALLOC(Op, DAG);
   case ISD::GET_DYNAMIC_AREA_OFFSET:
     return lowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
   case ISD::SMUL_LOHI:
     return lowerSMUL_LOHI(Op, DAG);
   case ISD::UMUL_LOHI:
     return lowerUMUL_LOHI(Op, DAG);
   case ISD::SDIVREM:
     return lowerSDIVREM(Op, DAG);
   case ISD::UDIVREM:
     return lowerUDIVREM(Op, DAG);
   case ISD::SADDO:
   case ISD::SSUBO:
   case ISD::UADDO:
   case ISD::USUBO:
     return lowerXALUO(Op, DAG);
   case ISD::ADDCARRY:
   case ISD::SUBCARRY:
     return lowerADDSUBCARRY(Op, DAG);
   case ISD::OR:
     return lowerOR(Op, DAG);
   case ISD::CTPOP:
     return lowerCTPOP(Op, DAG);
   case ISD::ATOMIC_FENCE:
     return lowerATOMIC_FENCE(Op, DAG);
   case ISD::ATOMIC_SWAP:
     return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_SWAPW);
   case ISD::ATOMIC_STORE:
     return lowerATOMIC_STORE(Op, DAG);
   case ISD::ATOMIC_LOAD:
     return lowerATOMIC_LOAD(Op, DAG);
   case ISD::ATOMIC_LOAD_ADD:
     return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD);
   case ISD::ATOMIC_LOAD_SUB:
     return lowerATOMIC_LOAD_SUB(Op, DAG);
   case ISD::ATOMIC_LOAD_AND:
     return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_AND);
   case ISD::ATOMIC_LOAD_OR:
     return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_OR);
   case ISD::ATOMIC_LOAD_XOR:
     return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_XOR);
   case ISD::ATOMIC_LOAD_NAND:
     return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_NAND);
   case ISD::ATOMIC_LOAD_MIN:
     return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MIN);
   case ISD::ATOMIC_LOAD_MAX:
     return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_MAX);
   case ISD::ATOMIC_LOAD_UMIN:
     return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMIN);
   case ISD::ATOMIC_LOAD_UMAX:
     return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_UMAX);
   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
     return lowerATOMIC_CMP_SWAP(Op, DAG);
   case ISD::STACKSAVE:
     return lowerSTACKSAVE(Op, DAG);
   case ISD::STACKRESTORE:
     return lowerSTACKRESTORE(Op, DAG);
   case ISD::PREFETCH:
     return lowerPREFETCH(Op, DAG);
   case ISD::INTRINSIC_W_CHAIN:
     return lowerINTRINSIC_W_CHAIN(Op, DAG);
   case ISD::INTRINSIC_WO_CHAIN:
     return lowerINTRINSIC_WO_CHAIN(Op, DAG);
   case ISD::BUILD_VECTOR:
     return lowerBUILD_VECTOR(Op, DAG);
   case ISD::VECTOR_SHUFFLE:
     return lowerVECTOR_SHUFFLE(Op, DAG);
   case ISD::SCALAR_TO_VECTOR:
     return lowerSCALAR_TO_VECTOR(Op, DAG);
   case ISD::INSERT_VECTOR_ELT:
     return lowerINSERT_VECTOR_ELT(Op, DAG);
   case ISD::EXTRACT_VECTOR_ELT:
     return lowerEXTRACT_VECTOR_ELT(Op, DAG);
   case ISD::SIGN_EXTEND_VECTOR_INREG:
     return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
   case ISD::ZERO_EXTEND_VECTOR_INREG:
     return lowerZERO_EXTEND_VECTOR_INREG(Op, DAG);
   case ISD::SHL:
     return lowerShift(Op, DAG, SystemZISD::VSHL_BY_SCALAR);
   case ISD::SRL:
     return lowerShift(Op, DAG, SystemZISD::VSRL_BY_SCALAR);
   case ISD::SRA:
     return lowerShift(Op, DAG, SystemZISD::VSRA_BY_SCALAR);
   default:
     llvm_unreachable("Unexpected node to lower");
   }
 }
 
 // Lower operations with invalid operand or result types (currently used
 // only for 128-bit integer types).
 void
 SystemZTargetLowering::LowerOperationWrapper(SDNode *N,
                                              SmallVectorImpl<SDValue> &Results,
                                              SelectionDAG &DAG) const {
   switch (N->getOpcode()) {
   case ISD::ATOMIC_LOAD: {
     SDLoc DL(N);
     SDVTList Tys = DAG.getVTList(MVT::Untyped, MVT::Other);
     SDValue Ops[] = { N->getOperand(0), N->getOperand(1) };
     MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
     SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_LOAD_128,
                                           DL, Tys, Ops, MVT::i128, MMO);
     Results.push_back(lowerGR128ToI128(DAG, Res));
     Results.push_back(Res.getValue(1));
     break;
   }
   case ISD::ATOMIC_STORE: {
     SDLoc DL(N);
     SDVTList Tys = DAG.getVTList(MVT::Other);
     SDValue Ops[] = { N->getOperand(0),
                       lowerI128ToGR128(DAG, N->getOperand(2)),
                       N->getOperand(1) };
     MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
     SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_STORE_128,
                                           DL, Tys, Ops, MVT::i128, MMO);
     // We have to enforce sequential consistency by performing a
     // serialization operation after the store.
     if (cast<AtomicSDNode>(N)->getSuccessOrdering() ==
         AtomicOrdering::SequentiallyConsistent)
       Res = SDValue(DAG.getMachineNode(SystemZ::Serialize, DL,
                                        MVT::Other, Res), 0);
     Results.push_back(Res);
     break;
   }
   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
     SDLoc DL(N);
     SDVTList Tys = DAG.getVTList(MVT::Untyped, MVT::i32, MVT::Other);
     SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
                       lowerI128ToGR128(DAG, N->getOperand(2)),
                       lowerI128ToGR128(DAG, N->getOperand(3)) };
     MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
     SDValue Res = DAG.getMemIntrinsicNode(SystemZISD::ATOMIC_CMP_SWAP_128,
                                           DL, Tys, Ops, MVT::i128, MMO);
     SDValue Success = emitSETCC(DAG, DL, Res.getValue(1),
                                 SystemZ::CCMASK_CS, SystemZ::CCMASK_CS_EQ);
     Success = DAG.getZExtOrTrunc(Success, DL, N->getValueType(1));
     Results.push_back(lowerGR128ToI128(DAG, Res));
     Results.push_back(Success);
     Results.push_back(Res.getValue(2));
     break;
   }
   case ISD::BITCAST: {
     SDValue Src = N->getOperand(0);
     if (N->getValueType(0) == MVT::i128 && Src.getValueType() == MVT::f128 &&
         !useSoftFloat()) {
       SDLoc DL(N);
       SDValue Lo, Hi;
       if (getRepRegClassFor(MVT::f128) == &SystemZ::VR128BitRegClass) {
         SDValue VecBC = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Src);
         Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, VecBC,
                          DAG.getConstant(1, DL, MVT::i32));
         Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, VecBC,
                          DAG.getConstant(0, DL, MVT::i32));
       } else {
         assert(getRepRegClassFor(MVT::f128) == &SystemZ::FP128BitRegClass &&
                "Unrecognized register class for f128.");
         SDValue LoFP = DAG.getTargetExtractSubreg(SystemZ::subreg_l64,
                                                   DL, MVT::f64, Src);
         SDValue HiFP = DAG.getTargetExtractSubreg(SystemZ::subreg_h64,
                                                   DL, MVT::f64, Src);
         Lo = DAG.getNode(ISD::BITCAST, DL, MVT::i64, LoFP);
         Hi = DAG.getNode(ISD::BITCAST, DL, MVT::i64, HiFP);
       }
       Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128, Lo, Hi));
     }
     break;
   }
   default:
     llvm_unreachable("Unexpected node to lower");
   }
 }
 
 void
 SystemZTargetLowering::ReplaceNodeResults(SDNode *N,
                                           SmallVectorImpl<SDValue> &Results,
                                           SelectionDAG &DAG) const {
   return LowerOperationWrapper(N, Results, DAG);
 }
 
 const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const {
 #define OPCODE(NAME) case SystemZISD::NAME: return "SystemZISD::" #NAME
   switch ((SystemZISD::NodeType)Opcode) {
     case SystemZISD::FIRST_NUMBER: break;
     OPCODE(RET_FLAG);
     OPCODE(CALL);
     OPCODE(SIBCALL);
     OPCODE(TLS_GDCALL);
     OPCODE(TLS_LDCALL);
     OPCODE(PCREL_WRAPPER);
     OPCODE(PCREL_OFFSET);
     OPCODE(ICMP);
     OPCODE(FCMP);
     OPCODE(STRICT_FCMP);
     OPCODE(STRICT_FCMPS);
     OPCODE(TM);
     OPCODE(BR_CCMASK);
     OPCODE(SELECT_CCMASK);
     OPCODE(ADJDYNALLOC);
     OPCODE(PROBED_ALLOCA);
     OPCODE(POPCNT);
     OPCODE(SMUL_LOHI);
     OPCODE(UMUL_LOHI);
     OPCODE(SDIVREM);
     OPCODE(UDIVREM);
     OPCODE(SADDO);
     OPCODE(SSUBO);
     OPCODE(UADDO);
     OPCODE(USUBO);
     OPCODE(ADDCARRY);
     OPCODE(SUBCARRY);
     OPCODE(GET_CCMASK);
     OPCODE(MVC);
     OPCODE(NC);
     OPCODE(OC);
     OPCODE(XC);
     OPCODE(CLC);
     OPCODE(MEMSET_MVC);
     OPCODE(STPCPY);
     OPCODE(STRCMP);
     OPCODE(SEARCH_STRING);
     OPCODE(IPM);
     OPCODE(MEMBARRIER);
     OPCODE(TBEGIN);
     OPCODE(TBEGIN_NOFLOAT);
     OPCODE(TEND);
     OPCODE(BYTE_MASK);
     OPCODE(ROTATE_MASK);
     OPCODE(REPLICATE);
     OPCODE(JOIN_DWORDS);
     OPCODE(SPLAT);
     OPCODE(MERGE_HIGH);
     OPCODE(MERGE_LOW);
     OPCODE(SHL_DOUBLE);
     OPCODE(PERMUTE_DWORDS);
     OPCODE(PERMUTE);
     OPCODE(PACK);
     OPCODE(PACKS_CC);
     OPCODE(PACKLS_CC);
     OPCODE(UNPACK_HIGH);
     OPCODE(UNPACKL_HIGH);
     OPCODE(UNPACK_LOW);
     OPCODE(UNPACKL_LOW);
     OPCODE(VSHL_BY_SCALAR);
     OPCODE(VSRL_BY_SCALAR);
     OPCODE(VSRA_BY_SCALAR);
     OPCODE(VSUM);
     OPCODE(VICMPE);
     OPCODE(VICMPH);
     OPCODE(VICMPHL);
     OPCODE(VICMPES);
     OPCODE(VICMPHS);
     OPCODE(VICMPHLS);
     OPCODE(VFCMPE);
     OPCODE(STRICT_VFCMPE);
     OPCODE(STRICT_VFCMPES);
     OPCODE(VFCMPH);
     OPCODE(STRICT_VFCMPH);
     OPCODE(STRICT_VFCMPHS);
     OPCODE(VFCMPHE);
     OPCODE(STRICT_VFCMPHE);
     OPCODE(STRICT_VFCMPHES);
     OPCODE(VFCMPES);
     OPCODE(VFCMPHS);
     OPCODE(VFCMPHES);
     OPCODE(VFTCI);
     OPCODE(VEXTEND);
     OPCODE(STRICT_VEXTEND);
     OPCODE(VROUND);
     OPCODE(STRICT_VROUND);
     OPCODE(VTM);
     OPCODE(VFAE_CC);
     OPCODE(VFAEZ_CC);
     OPCODE(VFEE_CC);
     OPCODE(VFEEZ_CC);
     OPCODE(VFENE_CC);
     OPCODE(VFENEZ_CC);
     OPCODE(VISTR_CC);
     OPCODE(VSTRC_CC);
     OPCODE(VSTRCZ_CC);
     OPCODE(VSTRS_CC);
     OPCODE(VSTRSZ_CC);
     OPCODE(TDC);
     OPCODE(ATOMIC_SWAPW);
     OPCODE(ATOMIC_LOADW_ADD);
     OPCODE(ATOMIC_LOADW_SUB);
     OPCODE(ATOMIC_LOADW_AND);
     OPCODE(ATOMIC_LOADW_OR);
     OPCODE(ATOMIC_LOADW_XOR);
     OPCODE(ATOMIC_LOADW_NAND);
     OPCODE(ATOMIC_LOADW_MIN);
     OPCODE(ATOMIC_LOADW_MAX);
     OPCODE(ATOMIC_LOADW_UMIN);
     OPCODE(ATOMIC_LOADW_UMAX);
     OPCODE(ATOMIC_CMP_SWAPW);
     OPCODE(ATOMIC_CMP_SWAP);
     OPCODE(ATOMIC_LOAD_128);
     OPCODE(ATOMIC_STORE_128);
     OPCODE(ATOMIC_CMP_SWAP_128);
     OPCODE(LRV);
     OPCODE(STRV);
     OPCODE(VLER);
     OPCODE(VSTER);
     OPCODE(PREFETCH);
   }
   return nullptr;
 #undef OPCODE
 }
 
 // Return true if VT is a vector whose elements are a whole number of bytes
 // in width. Also check for presence of vector support.
 bool SystemZTargetLowering::canTreatAsByteVector(EVT VT) const {
   if (!Subtarget.hasVector())
     return false;
 
   return VT.isVector() && VT.getScalarSizeInBits() % 8 == 0 && VT.isSimple();
 }
 
 // Try to simplify an EXTRACT_VECTOR_ELT from a vector of type VecVT
 // producing a result of type ResVT.  Op is a possibly bitcast version
 // of the input vector and Index is the index (based on type VecVT) that
 // should be extracted.  Return the new extraction if a simplification
 // was possible or if Force is true.
 SDValue SystemZTargetLowering::combineExtract(const SDLoc &DL, EVT ResVT,
                                               EVT VecVT, SDValue Op,
                                               unsigned Index,
                                               DAGCombinerInfo &DCI,
                                               bool Force) const {
   SelectionDAG &DAG = DCI.DAG;
 
   // The number of bytes being extracted.
   unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
 
   for (;;) {
     unsigned Opcode = Op.getOpcode();
     if (Opcode == ISD::BITCAST)
       // Look through bitcasts.
       Op = Op.getOperand(0);
     else if ((Opcode == ISD::VECTOR_SHUFFLE || Opcode == SystemZISD::SPLAT) &&
              canTreatAsByteVector(Op.getValueType())) {
       // Get a VPERM-like permute mask and see whether the bytes covered
       // by the extracted element are a contiguous sequence from one
       // source operand.
       SmallVector<int, SystemZ::VectorBytes> Bytes;
       if (!getVPermMask(Op, Bytes))
         break;
       int First;
       if (!getShuffleInput(Bytes, Index * BytesPerElement,
                            BytesPerElement, First))
         break;
       if (First < 0)
         return DAG.getUNDEF(ResVT);
       // Make sure the contiguous sequence starts at a multiple of the
       // original element size.
       unsigned Byte = unsigned(First) % Bytes.size();
       if (Byte % BytesPerElement != 0)
         break;
       // We can get the extracted value directly from an input.
       Index = Byte / BytesPerElement;
       Op = Op.getOperand(unsigned(First) / Bytes.size());
       Force = true;
     } else if (Opcode == ISD::BUILD_VECTOR &&
                canTreatAsByteVector(Op.getValueType())) {
       // We can only optimize this case if the BUILD_VECTOR elements are
       // at least as wide as the extracted value.
       EVT OpVT = Op.getValueType();
       unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
       if (OpBytesPerElement < BytesPerElement)
         break;
       // Make sure that the least-significant bit of the extracted value
       // is the least significant bit of an input.
       unsigned End = (Index + 1) * BytesPerElement;
       if (End % OpBytesPerElement != 0)
         break;
       // We're extracting the low part of one operand of the BUILD_VECTOR.
       Op = Op.getOperand(End / OpBytesPerElement - 1);
       if (!Op.getValueType().isInteger()) {
         EVT VT = MVT::getIntegerVT(Op.getValueSizeInBits());
         Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
         DCI.AddToWorklist(Op.getNode());
       }
       EVT VT = MVT::getIntegerVT(ResVT.getSizeInBits());
       Op = DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
       if (VT != ResVT) {
         DCI.AddToWorklist(Op.getNode());
         Op = DAG.getNode(ISD::BITCAST, DL, ResVT, Op);
       }
       return Op;
     } else if ((Opcode == ISD::SIGN_EXTEND_VECTOR_INREG ||
                 Opcode == ISD::ZERO_EXTEND_VECTOR_INREG ||
                 Opcode == ISD::ANY_EXTEND_VECTOR_INREG) &&
                canTreatAsByteVector(Op.getValueType()) &&
                canTreatAsByteVector(Op.getOperand(0).getValueType())) {
       // Make sure that only the unextended bits are significant.
       EVT ExtVT = Op.getValueType();
       EVT OpVT = Op.getOperand(0).getValueType();
       unsigned ExtBytesPerElement = ExtVT.getVectorElementType().getStoreSize();
       unsigned OpBytesPerElement = OpVT.getVectorElementType().getStoreSize();
       unsigned Byte = Index * BytesPerElement;
       unsigned SubByte = Byte % ExtBytesPerElement;
       unsigned MinSubByte = ExtBytesPerElement - OpBytesPerElement;
       if (SubByte < MinSubByte ||
           SubByte + BytesPerElement > ExtBytesPerElement)
         break;
       // Get the byte offset of the unextended element
       Byte = Byte / ExtBytesPerElement * OpBytesPerElement;
       // ...then add the byte offset relative to that element.
       Byte += SubByte - MinSubByte;
       if (Byte % BytesPerElement != 0)
         break;
       Op = Op.getOperand(0);
       Index = Byte / BytesPerElement;
       Force = true;
     } else
       break;
   }
   if (Force) {
     if (Op.getValueType() != VecVT) {
       Op = DAG.getNode(ISD::BITCAST, DL, VecVT, Op);
       DCI.AddToWorklist(Op.getNode());
     }
     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Op,
                        DAG.getConstant(Index, DL, MVT::i32));
   }
   return SDValue();
 }
 
 // Optimize vector operations in scalar value Op on the basis that Op
 // is truncated to TruncVT.
 SDValue SystemZTargetLowering::combineTruncateExtract(
     const SDLoc &DL, EVT TruncVT, SDValue Op, DAGCombinerInfo &DCI) const {
   // If we have (trunc (extract_vector_elt X, Y)), try to turn it into
   // (extract_vector_elt (bitcast X), Y'), where (bitcast X) has elements
   // of type TruncVT.
   if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
       TruncVT.getSizeInBits() % 8 == 0) {
     SDValue Vec = Op.getOperand(0);
     EVT VecVT = Vec.getValueType();
     if (canTreatAsByteVector(VecVT)) {
       if (auto *IndexN = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
         unsigned BytesPerElement = VecVT.getVectorElementType().getStoreSize();
         unsigned TruncBytes = TruncVT.getStoreSize();
         if (BytesPerElement % TruncBytes == 0) {
           // Calculate the value of Y' in the above description.  We are
           // splitting the original elements into Scale equal-sized pieces
           // and for truncation purposes want the last (least-significant)
           // of these pieces for IndexN.  This is easiest to do by calculating
           // the start index of the following element and then subtracting 1.
           unsigned Scale = BytesPerElement / TruncBytes;
           unsigned NewIndex = (IndexN->getZExtValue() + 1) * Scale - 1;
 
           // Defer the creation of the bitcast from X to combineExtract,
           // which might be able to optimize the extraction.
           VecVT = MVT::getVectorVT(MVT::getIntegerVT(TruncBytes * 8),
                                    VecVT.getStoreSize() / TruncBytes);
           EVT ResVT = (TruncBytes < 4 ? MVT::i32 : TruncVT);
           return combineExtract(DL, ResVT, VecVT, Vec, NewIndex, DCI, true);
         }
       }
     }
   }
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::combineZERO_EXTEND(
     SDNode *N, DAGCombinerInfo &DCI) const {
   // Convert (zext (select_ccmask C1, C2)) into (select_ccmask C1', C2')
   SelectionDAG &DAG = DCI.DAG;
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
   if (N0.getOpcode() == SystemZISD::SELECT_CCMASK) {
     auto *TrueOp = dyn_cast<ConstantSDNode>(N0.getOperand(0));
     auto *FalseOp = dyn_cast<ConstantSDNode>(N0.getOperand(1));
     if (TrueOp && FalseOp) {
       SDLoc DL(N0);
       SDValue Ops[] = { DAG.getConstant(TrueOp->getZExtValue(), DL, VT),
                         DAG.getConstant(FalseOp->getZExtValue(), DL, VT),
                         N0.getOperand(2), N0.getOperand(3), N0.getOperand(4) };
       SDValue NewSelect = DAG.getNode(SystemZISD::SELECT_CCMASK, DL, VT, Ops);
       // If N0 has multiple uses, change other uses as well.
       if (!N0.hasOneUse()) {
         SDValue TruncSelect =
           DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), NewSelect);
         DCI.CombineTo(N0.getNode(), TruncSelect);
       }
       return NewSelect;
     }
   }
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::combineSIGN_EXTEND_INREG(
     SDNode *N, DAGCombinerInfo &DCI) const {
   // Convert (sext_in_reg (setcc LHS, RHS, COND), i1)
   // and (sext_in_reg (any_extend (setcc LHS, RHS, COND)), i1)
   // into (select_cc LHS, RHS, -1, 0, COND)
   SelectionDAG &DAG = DCI.DAG;
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
   EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
   if (N0.hasOneUse() && N0.getOpcode() == ISD::ANY_EXTEND)
     N0 = N0.getOperand(0);
   if (EVT == MVT::i1 && N0.hasOneUse() && N0.getOpcode() == ISD::SETCC) {
     SDLoc DL(N0);
     SDValue Ops[] = { N0.getOperand(0), N0.getOperand(1),
                       DAG.getConstant(-1, DL, VT), DAG.getConstant(0, DL, VT),
                       N0.getOperand(2) };
     return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
   }
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::combineSIGN_EXTEND(
     SDNode *N, DAGCombinerInfo &DCI) const {
   // Convert (sext (ashr (shl X, C1), C2)) to
   // (ashr (shl (anyext X), C1'), C2')), since wider shifts are as
   // cheap as narrower ones.
   SelectionDAG &DAG = DCI.DAG;
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
   if (N0.hasOneUse() && N0.getOpcode() == ISD::SRA) {
     auto *SraAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1));
     SDValue Inner = N0.getOperand(0);
     if (SraAmt && Inner.hasOneUse() && Inner.getOpcode() == ISD::SHL) {
       if (auto *ShlAmt = dyn_cast<ConstantSDNode>(Inner.getOperand(1))) {
         unsigned Extra = (VT.getSizeInBits() - N0.getValueSizeInBits());
         unsigned NewShlAmt = ShlAmt->getZExtValue() + Extra;
         unsigned NewSraAmt = SraAmt->getZExtValue() + Extra;
         EVT ShiftVT = N0.getOperand(1).getValueType();
         SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, SDLoc(Inner), VT,
                                   Inner.getOperand(0));
         SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(Inner), VT, Ext,
                                   DAG.getConstant(NewShlAmt, SDLoc(Inner),
                                                   ShiftVT));
         return DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl,
                            DAG.getConstant(NewSraAmt, SDLoc(N0), ShiftVT));
       }
     }
   }
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::combineMERGE(
     SDNode *N, DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   unsigned Opcode = N->getOpcode();
   SDValue Op0 = N->getOperand(0);
   SDValue Op1 = N->getOperand(1);
   if (Op0.getOpcode() == ISD::BITCAST)
     Op0 = Op0.getOperand(0);
   if (ISD::isBuildVectorAllZeros(Op0.getNode())) {
     // (z_merge_* 0, 0) -> 0.  This is mostly useful for using VLLEZF
     // for v4f32.
     if (Op1 == N->getOperand(0))
       return Op1;
     // (z_merge_? 0, X) -> (z_unpackl_? 0, X).
     EVT VT = Op1.getValueType();
     unsigned ElemBytes = VT.getVectorElementType().getStoreSize();
     if (ElemBytes <= 4) {
       Opcode = (Opcode == SystemZISD::MERGE_HIGH ?
                 SystemZISD::UNPACKL_HIGH : SystemZISD::UNPACKL_LOW);
       EVT InVT = VT.changeVectorElementTypeToInteger();
       EVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(ElemBytes * 16),
                                    SystemZ::VectorBytes / ElemBytes / 2);
       if (VT != InVT) {
         Op1 = DAG.getNode(ISD::BITCAST, SDLoc(N), InVT, Op1);
         DCI.AddToWorklist(Op1.getNode());
       }
       SDValue Op = DAG.getNode(Opcode, SDLoc(N), OutVT, Op1);
       DCI.AddToWorklist(Op.getNode());
       return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
     }
   }
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::combineLOAD(
     SDNode *N, DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   EVT LdVT = N->getValueType(0);
   if (LdVT.isVector() || LdVT.isInteger())
     return SDValue();
   // Transform a scalar load that is REPLICATEd as well as having other
   // use(s) to the form where the other use(s) use the first element of the
   // REPLICATE instead of the load. Otherwise instruction selection will not
   // produce a VLREP. Avoid extracting to a GPR, so only do this for floating
   // point loads.
 
   SDValue Replicate;
   SmallVector<SDNode*, 8> OtherUses;
   for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
        UI != UE; ++UI) {
     if (UI->getOpcode() == SystemZISD::REPLICATE) {
       if (Replicate)
         return SDValue(); // Should never happen
       Replicate = SDValue(*UI, 0);
     }
     else if (UI.getUse().getResNo() == 0)
       OtherUses.push_back(*UI);
   }
   if (!Replicate || OtherUses.empty())
     return SDValue();
 
   SDLoc DL(N);
   SDValue Extract0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, LdVT,
                               Replicate, DAG.getConstant(0, DL, MVT::i32));
   // Update uses of the loaded Value while preserving old chains.
   for (SDNode *U : OtherUses) {
     SmallVector<SDValue, 8> Ops;
     for (SDValue Op : U->ops())
       Ops.push_back((Op.getNode() == N && Op.getResNo() == 0) ? Extract0 : Op);
     DAG.UpdateNodeOperands(U, Ops);
   }
   return SDValue(N, 0);
 }
 
 bool SystemZTargetLowering::canLoadStoreByteSwapped(EVT VT) const {
   if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64)
     return true;
   if (Subtarget.hasVectorEnhancements2())
     if (VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v2i64)
       return true;
   return false;
 }
 
 static bool isVectorElementSwap(ArrayRef<int> M, EVT VT) {
   if (!VT.isVector() || !VT.isSimple() ||
       VT.getSizeInBits() != 128 ||
       VT.getScalarSizeInBits() % 8 != 0)
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
   for (unsigned i = 0; i < NumElts; ++i) {
     if (M[i] < 0) continue; // ignore UNDEF indices
     if ((unsigned) M[i] != NumElts - 1 - i)
       return false;
   }
 
   return true;
 }
 
 SDValue SystemZTargetLowering::combineSTORE(
     SDNode *N, DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   auto *SN = cast<StoreSDNode>(N);
   auto &Op1 = N->getOperand(1);
   EVT MemVT = SN->getMemoryVT();
   // If we have (truncstoreiN (extract_vector_elt X, Y), Z) then it is better
   // for the extraction to be done on a vMiN value, so that we can use VSTE.
   // If X has wider elements then convert it to:
   // (truncstoreiN (extract_vector_elt (bitcast X), Y2), Z).
   if (MemVT.isInteger() && SN->isTruncatingStore()) {
     if (SDValue Value =
             combineTruncateExtract(SDLoc(N), MemVT, SN->getValue(), DCI)) {
       DCI.AddToWorklist(Value.getNode());
 
       // Rewrite the store with the new form of stored value.
       return DAG.getTruncStore(SN->getChain(), SDLoc(SN), Value,
                                SN->getBasePtr(), SN->getMemoryVT(),
                                SN->getMemOperand());
     }
   }
   // Combine STORE (BSWAP) into STRVH/STRV/STRVG/VSTBR
   if (!SN->isTruncatingStore() &&
       Op1.getOpcode() == ISD::BSWAP &&
       Op1.getNode()->hasOneUse() &&
       canLoadStoreByteSwapped(Op1.getValueType())) {
 
       SDValue BSwapOp = Op1.getOperand(0);
 
       if (BSwapOp.getValueType() == MVT::i16)
         BSwapOp = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), MVT::i32, BSwapOp);
 
       SDValue Ops[] = {
         N->getOperand(0), BSwapOp, N->getOperand(2)
       };
 
       return
         DAG.getMemIntrinsicNode(SystemZISD::STRV, SDLoc(N), DAG.getVTList(MVT::Other),
                                 Ops, MemVT, SN->getMemOperand());
     }
   // Combine STORE (element-swap) into VSTER
   if (!SN->isTruncatingStore() &&
       Op1.getOpcode() == ISD::VECTOR_SHUFFLE &&
       Op1.getNode()->hasOneUse() &&
       Subtarget.hasVectorEnhancements2()) {
     ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op1.getNode());
     ArrayRef<int> ShuffleMask = SVN->getMask();
     if (isVectorElementSwap(ShuffleMask, Op1.getValueType())) {
       SDValue Ops[] = {
         N->getOperand(0), Op1.getOperand(0), N->getOperand(2)
       };
 
       return DAG.getMemIntrinsicNode(SystemZISD::VSTER, SDLoc(N),
                                      DAG.getVTList(MVT::Other),
                                      Ops, MemVT, SN->getMemOperand());
     }
   }
 
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::combineVECTOR_SHUFFLE(
     SDNode *N, DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   // Combine element-swap (LOAD) into VLER
   if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) &&
       N->getOperand(0).hasOneUse() &&
       Subtarget.hasVectorEnhancements2()) {
     ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
     ArrayRef<int> ShuffleMask = SVN->getMask();
     if (isVectorElementSwap(ShuffleMask, N->getValueType(0))) {
       SDValue Load = N->getOperand(0);
       LoadSDNode *LD = cast<LoadSDNode>(Load);
 
       // Create the element-swapping load.
       SDValue Ops[] = {
         LD->getChain(),    // Chain
         LD->getBasePtr()   // Ptr
       };
       SDValue ESLoad =
         DAG.getMemIntrinsicNode(SystemZISD::VLER, SDLoc(N),
                                 DAG.getVTList(LD->getValueType(0), MVT::Other),
                                 Ops, LD->getMemoryVT(), LD->getMemOperand());
 
       // First, combine the VECTOR_SHUFFLE away.  This makes the value produced
       // by the load dead.
       DCI.CombineTo(N, ESLoad);
 
       // Next, combine the load away, we give it a bogus result value but a real
       // chain result.  The result value is dead because the shuffle is dead.
       DCI.CombineTo(Load.getNode(), ESLoad, ESLoad.getValue(1));
 
       // Return N so it doesn't get rechecked!
       return SDValue(N, 0);
     }
   }
 
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::combineEXTRACT_VECTOR_ELT(
     SDNode *N, DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
 
   if (!Subtarget.hasVector())
     return SDValue();
 
   // Look through bitcasts that retain the number of vector elements.
   SDValue Op = N->getOperand(0);
   if (Op.getOpcode() == ISD::BITCAST &&
       Op.getValueType().isVector() &&
       Op.getOperand(0).getValueType().isVector() &&
       Op.getValueType().getVectorNumElements() ==
       Op.getOperand(0).getValueType().getVectorNumElements())
     Op = Op.getOperand(0);
 
   // Pull BSWAP out of a vector extraction.
   if (Op.getOpcode() == ISD::BSWAP && Op.hasOneUse()) {
     EVT VecVT = Op.getValueType();
     EVT EltVT = VecVT.getVectorElementType();
     Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), EltVT,
                      Op.getOperand(0), N->getOperand(1));
     DCI.AddToWorklist(Op.getNode());
     Op = DAG.getNode(ISD::BSWAP, SDLoc(N), EltVT, Op);
     if (EltVT != N->getValueType(0)) {
       DCI.AddToWorklist(Op.getNode());
       Op = DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op);
     }
     return Op;
   }
 
   // Try to simplify a vector extraction.
   if (auto *IndexN = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
     SDValue Op0 = N->getOperand(0);
     EVT VecVT = Op0.getValueType();
     return combineExtract(SDLoc(N), N->getValueType(0), VecVT, Op0,
                           IndexN->getZExtValue(), DCI, false);
   }
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::combineJOIN_DWORDS(
     SDNode *N, DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   // (join_dwords X, X) == (replicate X)
   if (N->getOperand(0) == N->getOperand(1))
     return DAG.getNode(SystemZISD::REPLICATE, SDLoc(N), N->getValueType(0),
                        N->getOperand(0));
   return SDValue();
 }
 
 static SDValue MergeInputChains(SDNode *N1, SDNode *N2) {
   SDValue Chain1 = N1->getOperand(0);
   SDValue Chain2 = N2->getOperand(0);
 
   // Trivial case: both nodes take the same chain.
   if (Chain1 == Chain2)
     return Chain1;
 
   // FIXME - we could handle more complex cases via TokenFactor,
   // assuming we can verify that this would not create a cycle.
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::combineFP_ROUND(
     SDNode *N, DAGCombinerInfo &DCI) const {
 
   if (!Subtarget.hasVector())
     return SDValue();
 
   // (fpround (extract_vector_elt X 0))
   // (fpround (extract_vector_elt X 1)) ->
   // (extract_vector_elt (VROUND X) 0)
   // (extract_vector_elt (VROUND X) 2)
   //
   // This is a special case since the target doesn't really support v2f32s.
   unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
   SelectionDAG &DAG = DCI.DAG;
   SDValue Op0 = N->getOperand(OpNo);
   if (N->getValueType(0) == MVT::f32 &&
       Op0.hasOneUse() &&
       Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
       Op0.getOperand(0).getValueType() == MVT::v2f64 &&
       Op0.getOperand(1).getOpcode() == ISD::Constant &&
       cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue() == 0) {
     SDValue Vec = Op0.getOperand(0);
     for (auto *U : Vec->uses()) {
       if (U != Op0.getNode() &&
           U->hasOneUse() &&
           U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
           U->getOperand(0) == Vec &&
           U->getOperand(1).getOpcode() == ISD::Constant &&
           cast<ConstantSDNode>(U->getOperand(1))->getZExtValue() == 1) {
         SDValue OtherRound = SDValue(*U->use_begin(), 0);
         if (OtherRound.getOpcode() == N->getOpcode() &&
             OtherRound.getOperand(OpNo) == SDValue(U, 0) &&
             OtherRound.getValueType() == MVT::f32) {
           SDValue VRound, Chain;
           if (N->isStrictFPOpcode()) {
             Chain = MergeInputChains(N, OtherRound.getNode());
             if (!Chain)
               continue;
             VRound = DAG.getNode(SystemZISD::STRICT_VROUND, SDLoc(N),
                                  {MVT::v4f32, MVT::Other}, {Chain, Vec});
             Chain = VRound.getValue(1);
           } else
             VRound = DAG.getNode(SystemZISD::VROUND, SDLoc(N),
                                  MVT::v4f32, Vec);
           DCI.AddToWorklist(VRound.getNode());
           SDValue Extract1 =
             DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(U), MVT::f32,
                         VRound, DAG.getConstant(2, SDLoc(U), MVT::i32));
           DCI.AddToWorklist(Extract1.getNode());
           DAG.ReplaceAllUsesOfValueWith(OtherRound, Extract1);
           if (Chain)
             DAG.ReplaceAllUsesOfValueWith(OtherRound.getValue(1), Chain);
           SDValue Extract0 =
             DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op0), MVT::f32,
                         VRound, DAG.getConstant(0, SDLoc(Op0), MVT::i32));
           if (Chain)
             return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op0),
                                N->getVTList(), Extract0, Chain);
           return Extract0;
         }
       }
     }
   }
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::combineFP_EXTEND(
     SDNode *N, DAGCombinerInfo &DCI) const {
 
   if (!Subtarget.hasVector())
     return SDValue();
 
   // (fpextend (extract_vector_elt X 0))
   // (fpextend (extract_vector_elt X 2)) ->
   // (extract_vector_elt (VEXTEND X) 0)
   // (extract_vector_elt (VEXTEND X) 1)
   //
   // This is a special case since the target doesn't really support v2f32s.
   unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
   SelectionDAG &DAG = DCI.DAG;
   SDValue Op0 = N->getOperand(OpNo);
   if (N->getValueType(0) == MVT::f64 &&
       Op0.hasOneUse() &&
       Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
       Op0.getOperand(0).getValueType() == MVT::v4f32 &&
       Op0.getOperand(1).getOpcode() == ISD::Constant &&
       cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue() == 0) {
     SDValue Vec = Op0.getOperand(0);
     for (auto *U : Vec->uses()) {
       if (U != Op0.getNode() &&
           U->hasOneUse() &&
           U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
           U->getOperand(0) == Vec &&
           U->getOperand(1).getOpcode() == ISD::Constant &&
           cast<ConstantSDNode>(U->getOperand(1))->getZExtValue() == 2) {
         SDValue OtherExtend = SDValue(*U->use_begin(), 0);
         if (OtherExtend.getOpcode() == N->getOpcode() &&
             OtherExtend.getOperand(OpNo) == SDValue(U, 0) &&
             OtherExtend.getValueType() == MVT::f64) {
           SDValue VExtend, Chain;
           if (N->isStrictFPOpcode()) {
             Chain = MergeInputChains(N, OtherExtend.getNode());
             if (!Chain)
               continue;
             VExtend = DAG.getNode(SystemZISD::STRICT_VEXTEND, SDLoc(N),
                                   {MVT::v2f64, MVT::Other}, {Chain, Vec});
             Chain = VExtend.getValue(1);
           } else
             VExtend = DAG.getNode(SystemZISD::VEXTEND, SDLoc(N),
                                   MVT::v2f64, Vec);
           DCI.AddToWorklist(VExtend.getNode());
           SDValue Extract1 =
             DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(U), MVT::f64,
                         VExtend, DAG.getConstant(1, SDLoc(U), MVT::i32));
           DCI.AddToWorklist(Extract1.getNode());
           DAG.ReplaceAllUsesOfValueWith(OtherExtend, Extract1);
           if (Chain)
             DAG.ReplaceAllUsesOfValueWith(OtherExtend.getValue(1), Chain);
           SDValue Extract0 =
             DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op0), MVT::f64,
                         VExtend, DAG.getConstant(0, SDLoc(Op0), MVT::i32));
           if (Chain)
             return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op0),
                                N->getVTList(), Extract0, Chain);
           return Extract0;
         }
       }
     }
   }
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::combineINT_TO_FP(
     SDNode *N, DAGCombinerInfo &DCI) const {
   if (DCI.Level != BeforeLegalizeTypes)
     return SDValue();
+  SelectionDAG &DAG = DCI.DAG;
+  LLVMContext &Ctx = *DAG.getContext();
   unsigned Opcode = N->getOpcode();
   EVT OutVT = N->getValueType(0);
-  SelectionDAG &DAG = DCI.DAG;
+  Type *OutLLVMTy = OutVT.getTypeForEVT(Ctx);
   SDValue Op = N->getOperand(0);
-  unsigned OutScalarBits = OutVT.getScalarSizeInBits();
+  unsigned OutScalarBits = OutLLVMTy->getScalarSizeInBits();
   unsigned InScalarBits = Op->getValueType(0).getScalarSizeInBits();
 
   // Insert an extension before type-legalization to avoid scalarization, e.g.:
   // v2f64 = uint_to_fp v2i16
   // =>
   // v2f64 = uint_to_fp (v2i64 zero_extend v2i16)
-  if (OutVT.isVector() && OutScalarBits > InScalarBits) {
-    MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(OutVT.getScalarSizeInBits()),
-                                 OutVT.getVectorNumElements());
+  if (OutLLVMTy->isVectorTy() && OutScalarBits > InScalarBits &&
+      OutScalarBits <= 64) {
+    unsigned NumElts = cast<FixedVectorType>(OutLLVMTy)->getNumElements();
+    EVT ExtVT = EVT::getVectorVT(
+        Ctx, EVT::getIntegerVT(Ctx, OutLLVMTy->getScalarSizeInBits()), NumElts);
     unsigned ExtOpcode =
-      (Opcode == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND);
+        (Opcode == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND);
     SDValue ExtOp = DAG.getNode(ExtOpcode, SDLoc(N), ExtVT, Op);
     return DAG.getNode(Opcode, SDLoc(N), OutVT, ExtOp);
   }
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::combineBSWAP(
     SDNode *N, DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   // Combine BSWAP (LOAD) into LRVH/LRV/LRVG/VLBR
   if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) &&
       N->getOperand(0).hasOneUse() &&
       canLoadStoreByteSwapped(N->getValueType(0))) {
       SDValue Load = N->getOperand(0);
       LoadSDNode *LD = cast<LoadSDNode>(Load);
 
       // Create the byte-swapping load.
       SDValue Ops[] = {
         LD->getChain(),    // Chain
         LD->getBasePtr()   // Ptr
       };
       EVT LoadVT = N->getValueType(0);
       if (LoadVT == MVT::i16)
         LoadVT = MVT::i32;
       SDValue BSLoad =
         DAG.getMemIntrinsicNode(SystemZISD::LRV, SDLoc(N),
                                 DAG.getVTList(LoadVT, MVT::Other),
                                 Ops, LD->getMemoryVT(), LD->getMemOperand());
 
       // If this is an i16 load, insert the truncate.
       SDValue ResVal = BSLoad;
       if (N->getValueType(0) == MVT::i16)
         ResVal = DAG.getNode(ISD::TRUNCATE, SDLoc(N), MVT::i16, BSLoad);
 
       // First, combine the bswap away.  This makes the value produced by the
       // load dead.
       DCI.CombineTo(N, ResVal);
 
       // Next, combine the load away, we give it a bogus result value but a real
       // chain result.  The result value is dead because the bswap is dead.
       DCI.CombineTo(Load.getNode(), ResVal, BSLoad.getValue(1));
 
       // Return N so it doesn't get rechecked!
       return SDValue(N, 0);
     }
 
   // Look through bitcasts that retain the number of vector elements.
   SDValue Op = N->getOperand(0);
   if (Op.getOpcode() == ISD::BITCAST &&
       Op.getValueType().isVector() &&
       Op.getOperand(0).getValueType().isVector() &&
       Op.getValueType().getVectorNumElements() ==
       Op.getOperand(0).getValueType().getVectorNumElements())
     Op = Op.getOperand(0);
 
   // Push BSWAP into a vector insertion if at least one side then simplifies.
   if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT && Op.hasOneUse()) {
     SDValue Vec = Op.getOperand(0);
     SDValue Elt = Op.getOperand(1);
     SDValue Idx = Op.getOperand(2);
 
     if (DAG.isConstantIntBuildVectorOrConstantInt(Vec) ||
         Vec.getOpcode() == ISD::BSWAP || Vec.isUndef() ||
         DAG.isConstantIntBuildVectorOrConstantInt(Elt) ||
         Elt.getOpcode() == ISD::BSWAP || Elt.isUndef() ||
         (canLoadStoreByteSwapped(N->getValueType(0)) &&
          ISD::isNON_EXTLoad(Elt.getNode()) && Elt.hasOneUse())) {
       EVT VecVT = N->getValueType(0);
       EVT EltVT = N->getValueType(0).getVectorElementType();
       if (VecVT != Vec.getValueType()) {
         Vec = DAG.getNode(ISD::BITCAST, SDLoc(N), VecVT, Vec);
         DCI.AddToWorklist(Vec.getNode());
       }
       if (EltVT != Elt.getValueType()) {
         Elt = DAG.getNode(ISD::BITCAST, SDLoc(N), EltVT, Elt);
         DCI.AddToWorklist(Elt.getNode());
       }
       Vec = DAG.getNode(ISD::BSWAP, SDLoc(N), VecVT, Vec);
       DCI.AddToWorklist(Vec.getNode());
       Elt = DAG.getNode(ISD::BSWAP, SDLoc(N), EltVT, Elt);
       DCI.AddToWorklist(Elt.getNode());
       return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), VecVT,
                          Vec, Elt, Idx);
     }
   }
 
   // Push BSWAP into a vector shuffle if at least one side then simplifies.
   ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(Op);
   if (SV && Op.hasOneUse()) {
     SDValue Op0 = Op.getOperand(0);
     SDValue Op1 = Op.getOperand(1);
 
     if (DAG.isConstantIntBuildVectorOrConstantInt(Op0) ||
         Op0.getOpcode() == ISD::BSWAP || Op0.isUndef() ||
         DAG.isConstantIntBuildVectorOrConstantInt(Op1) ||
         Op1.getOpcode() == ISD::BSWAP || Op1.isUndef()) {
       EVT VecVT = N->getValueType(0);
       if (VecVT != Op0.getValueType()) {
         Op0 = DAG.getNode(ISD::BITCAST, SDLoc(N), VecVT, Op0);
         DCI.AddToWorklist(Op0.getNode());
       }
       if (VecVT != Op1.getValueType()) {
         Op1 = DAG.getNode(ISD::BITCAST, SDLoc(N), VecVT, Op1);
         DCI.AddToWorklist(Op1.getNode());
       }
       Op0 = DAG.getNode(ISD::BSWAP, SDLoc(N), VecVT, Op0);
       DCI.AddToWorklist(Op0.getNode());
       Op1 = DAG.getNode(ISD::BSWAP, SDLoc(N), VecVT, Op1);
       DCI.AddToWorklist(Op1.getNode());
       return DAG.getVectorShuffle(VecVT, SDLoc(N), Op0, Op1, SV->getMask());
     }
   }
 
   return SDValue();
 }
 
 static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask) {
   // We have a SELECT_CCMASK or BR_CCMASK comparing the condition code
   // set by the CCReg instruction using the CCValid / CCMask masks,
   // If the CCReg instruction is itself a ICMP testing the condition
   // code set by some other instruction, see whether we can directly
   // use that condition code.
 
   // Verify that we have an ICMP against some constant.
   if (CCValid != SystemZ::CCMASK_ICMP)
     return false;
   auto *ICmp = CCReg.getNode();
   if (ICmp->getOpcode() != SystemZISD::ICMP)
     return false;
   auto *CompareLHS = ICmp->getOperand(0).getNode();
   auto *CompareRHS = dyn_cast<ConstantSDNode>(ICmp->getOperand(1));
   if (!CompareRHS)
     return false;
 
   // Optimize the case where CompareLHS is a SELECT_CCMASK.
   if (CompareLHS->getOpcode() == SystemZISD::SELECT_CCMASK) {
     // Verify that we have an appropriate mask for a EQ or NE comparison.
     bool Invert = false;
     if (CCMask == SystemZ::CCMASK_CMP_NE)
       Invert = !Invert;
     else if (CCMask != SystemZ::CCMASK_CMP_EQ)
       return false;
 
     // Verify that the ICMP compares against one of select values.
     auto *TrueVal = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(0));
     if (!TrueVal)
       return false;
     auto *FalseVal = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(1));
     if (!FalseVal)
       return false;
     if (CompareRHS->getZExtValue() == FalseVal->getZExtValue())
       Invert = !Invert;
     else if (CompareRHS->getZExtValue() != TrueVal->getZExtValue())
       return false;
 
     // Compute the effective CC mask for the new branch or select.
     auto *NewCCValid = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(2));
     auto *NewCCMask = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(3));
     if (!NewCCValid || !NewCCMask)
       return false;
     CCValid = NewCCValid->getZExtValue();
     CCMask = NewCCMask->getZExtValue();
     if (Invert)
       CCMask ^= CCValid;
 
     // Return the updated CCReg link.
     CCReg = CompareLHS->getOperand(4);
     return true;
   }
 
   // Optimize the case where CompareRHS is (SRA (SHL (IPM))).
   if (CompareLHS->getOpcode() == ISD::SRA) {
     auto *SRACount = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(1));
     if (!SRACount || SRACount->getZExtValue() != 30)
       return false;
     auto *SHL = CompareLHS->getOperand(0).getNode();
     if (SHL->getOpcode() != ISD::SHL)
       return false;
     auto *SHLCount = dyn_cast<ConstantSDNode>(SHL->getOperand(1));
     if (!SHLCount || SHLCount->getZExtValue() != 30 - SystemZ::IPM_CC)
       return false;
     auto *IPM = SHL->getOperand(0).getNode();
     if (IPM->getOpcode() != SystemZISD::IPM)
       return false;
 
     // Avoid introducing CC spills (because SRA would clobber CC).
     if (!CompareLHS->hasOneUse())
       return false;
     // Verify that the ICMP compares against zero.
     if (CompareRHS->getZExtValue() != 0)
       return false;
 
     // Compute the effective CC mask for the new branch or select.
     CCMask = SystemZ::reverseCCMask(CCMask);
 
     // Return the updated CCReg link.
     CCReg = IPM->getOperand(0);
     return true;
   }
 
   return false;
 }
 
 SDValue SystemZTargetLowering::combineBR_CCMASK(
     SDNode *N, DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
 
   // Combine BR_CCMASK (ICMP (SELECT_CCMASK)) into a single BR_CCMASK.
   auto *CCValid = dyn_cast<ConstantSDNode>(N->getOperand(1));
   auto *CCMask = dyn_cast<ConstantSDNode>(N->getOperand(2));
   if (!CCValid || !CCMask)
     return SDValue();
 
   int CCValidVal = CCValid->getZExtValue();
   int CCMaskVal = CCMask->getZExtValue();
   SDValue Chain = N->getOperand(0);
   SDValue CCReg = N->getOperand(4);
 
   if (combineCCMask(CCReg, CCValidVal, CCMaskVal))
     return DAG.getNode(SystemZISD::BR_CCMASK, SDLoc(N), N->getValueType(0),
                        Chain,
                        DAG.getTargetConstant(CCValidVal, SDLoc(N), MVT::i32),
                        DAG.getTargetConstant(CCMaskVal, SDLoc(N), MVT::i32),
                        N->getOperand(3), CCReg);
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::combineSELECT_CCMASK(
     SDNode *N, DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
 
   // Combine SELECT_CCMASK (ICMP (SELECT_CCMASK)) into a single SELECT_CCMASK.
   auto *CCValid = dyn_cast<ConstantSDNode>(N->getOperand(2));
   auto *CCMask = dyn_cast<ConstantSDNode>(N->getOperand(3));
   if (!CCValid || !CCMask)
     return SDValue();
 
   int CCValidVal = CCValid->getZExtValue();
   int CCMaskVal = CCMask->getZExtValue();
   SDValue CCReg = N->getOperand(4);
 
   if (combineCCMask(CCReg, CCValidVal, CCMaskVal))
     return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
                        N->getOperand(0), N->getOperand(1),
                        DAG.getTargetConstant(CCValidVal, SDLoc(N), MVT::i32),
                        DAG.getTargetConstant(CCMaskVal, SDLoc(N), MVT::i32),
                        CCReg);
   return SDValue();
 }
 
 
 SDValue SystemZTargetLowering::combineGET_CCMASK(
     SDNode *N, DAGCombinerInfo &DCI) const {
 
   // Optimize away GET_CCMASK (SELECT_CCMASK) if the CC masks are compatible
   auto *CCValid = dyn_cast<ConstantSDNode>(N->getOperand(1));
   auto *CCMask = dyn_cast<ConstantSDNode>(N->getOperand(2));
   if (!CCValid || !CCMask)
     return SDValue();
   int CCValidVal = CCValid->getZExtValue();
   int CCMaskVal = CCMask->getZExtValue();
 
   SDValue Select = N->getOperand(0);
   if (Select->getOpcode() != SystemZISD::SELECT_CCMASK)
     return SDValue();
 
   auto *SelectCCValid = dyn_cast<ConstantSDNode>(Select->getOperand(2));
   auto *SelectCCMask = dyn_cast<ConstantSDNode>(Select->getOperand(3));
   if (!SelectCCValid || !SelectCCMask)
     return SDValue();
   int SelectCCValidVal = SelectCCValid->getZExtValue();
   int SelectCCMaskVal = SelectCCMask->getZExtValue();
 
   auto *TrueVal = dyn_cast<ConstantSDNode>(Select->getOperand(0));
   auto *FalseVal = dyn_cast<ConstantSDNode>(Select->getOperand(1));
   if (!TrueVal || !FalseVal)
     return SDValue();
   if (TrueVal->getZExtValue() != 0 && FalseVal->getZExtValue() == 0)
     ;
   else if (TrueVal->getZExtValue() == 0 && FalseVal->getZExtValue() != 0)
     SelectCCMaskVal ^= SelectCCValidVal;
   else
     return SDValue();
 
   if (SelectCCValidVal & ~CCValidVal)
     return SDValue();
   if (SelectCCMaskVal != (CCMaskVal & SelectCCValidVal))
     return SDValue();
 
   return Select->getOperand(4);
 }
 
 SDValue SystemZTargetLowering::combineIntDIVREM(
     SDNode *N, DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
   EVT VT = N->getValueType(0);
   // In the case where the divisor is a vector of constants a cheaper
   // sequence of instructions can replace the divide. BuildSDIV is called to
   // do this during DAG combining, but it only succeeds when it can build a
   // multiplication node. The only option for SystemZ is ISD::SMUL_LOHI, and
   // since it is not Legal but Custom it can only happen before
   // legalization. Therefore we must scalarize this early before Combine
   // 1. For widened vectors, this is already the result of type legalization.
   if (DCI.Level == BeforeLegalizeTypes && VT.isVector() && isTypeLegal(VT) &&
       DAG.isConstantIntBuildVectorOrConstantInt(N->getOperand(1)))
     return DAG.UnrollVectorOp(N);
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::combineINTRINSIC(
     SDNode *N, DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
 
   unsigned Id = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
   switch (Id) {
   // VECTOR LOAD (RIGHTMOST) WITH LENGTH with a length operand of 15
   // or larger is simply a vector load.
   case Intrinsic::s390_vll:
   case Intrinsic::s390_vlrl:
     if (auto *C = dyn_cast<ConstantSDNode>(N->getOperand(2)))
       if (C->getZExtValue() >= 15)
         return DAG.getLoad(N->getValueType(0), SDLoc(N), N->getOperand(0),
                            N->getOperand(3), MachinePointerInfo());
     break;
   // Likewise for VECTOR STORE (RIGHTMOST) WITH LENGTH.
   case Intrinsic::s390_vstl:
   case Intrinsic::s390_vstrl:
     if (auto *C = dyn_cast<ConstantSDNode>(N->getOperand(3)))
       if (C->getZExtValue() >= 15)
         return DAG.getStore(N->getOperand(0), SDLoc(N), N->getOperand(2),
                             N->getOperand(4), MachinePointerInfo());
     break;
   }
 
   return SDValue();
 }
 
 SDValue SystemZTargetLowering::unwrapAddress(SDValue N) const {
   if (N->getOpcode() == SystemZISD::PCREL_WRAPPER)
     return N->getOperand(0);
   return N;
 }
 
 SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
                                                  DAGCombinerInfo &DCI) const {
   switch(N->getOpcode()) {
   default: break;
   case ISD::ZERO_EXTEND:        return combineZERO_EXTEND(N, DCI);
   case ISD::SIGN_EXTEND:        return combineSIGN_EXTEND(N, DCI);
   case ISD::SIGN_EXTEND_INREG:  return combineSIGN_EXTEND_INREG(N, DCI);
   case SystemZISD::MERGE_HIGH:
   case SystemZISD::MERGE_LOW:   return combineMERGE(N, DCI);
   case ISD::LOAD:               return combineLOAD(N, DCI);
   case ISD::STORE:              return combineSTORE(N, DCI);
   case ISD::VECTOR_SHUFFLE:     return combineVECTOR_SHUFFLE(N, DCI);
   case ISD::EXTRACT_VECTOR_ELT: return combineEXTRACT_VECTOR_ELT(N, DCI);
   case SystemZISD::JOIN_DWORDS: return combineJOIN_DWORDS(N, DCI);
   case ISD::STRICT_FP_ROUND:
   case ISD::FP_ROUND:           return combineFP_ROUND(N, DCI);
   case ISD::STRICT_FP_EXTEND:
   case ISD::FP_EXTEND:          return combineFP_EXTEND(N, DCI);
   case ISD::SINT_TO_FP:
   case ISD::UINT_TO_FP:         return combineINT_TO_FP(N, DCI);
   case ISD::BSWAP:              return combineBSWAP(N, DCI);
   case SystemZISD::BR_CCMASK:   return combineBR_CCMASK(N, DCI);
   case SystemZISD::SELECT_CCMASK: return combineSELECT_CCMASK(N, DCI);
   case SystemZISD::GET_CCMASK:  return combineGET_CCMASK(N, DCI);
   case ISD::SDIV:
   case ISD::UDIV:
   case ISD::SREM:
   case ISD::UREM:               return combineIntDIVREM(N, DCI);
   case ISD::INTRINSIC_W_CHAIN:
   case ISD::INTRINSIC_VOID:     return combineINTRINSIC(N, DCI);
   }
 
   return SDValue();
 }
 
 // Return the demanded elements for the OpNo source operand of Op. DemandedElts
 // are for Op.
 static APInt getDemandedSrcElements(SDValue Op, const APInt &DemandedElts,
                                     unsigned OpNo) {
   EVT VT = Op.getValueType();
   unsigned NumElts = (VT.isVector() ? VT.getVectorNumElements() : 1);
   APInt SrcDemE;
   unsigned Opcode = Op.getOpcode();
   if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
     unsigned Id = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
     switch (Id) {
     case Intrinsic::s390_vpksh:   // PACKS
     case Intrinsic::s390_vpksf:
     case Intrinsic::s390_vpksg:
     case Intrinsic::s390_vpkshs:  // PACKS_CC
     case Intrinsic::s390_vpksfs:
     case Intrinsic::s390_vpksgs:
     case Intrinsic::s390_vpklsh:  // PACKLS
     case Intrinsic::s390_vpklsf:
     case Intrinsic::s390_vpklsg:
     case Intrinsic::s390_vpklshs: // PACKLS_CC
     case Intrinsic::s390_vpklsfs:
     case Intrinsic::s390_vpklsgs:
       // VECTOR PACK truncates the elements of two source vectors into one.
       SrcDemE = DemandedElts;
       if (OpNo == 2)
         SrcDemE.lshrInPlace(NumElts / 2);
       SrcDemE = SrcDemE.trunc(NumElts / 2);
       break;
       // VECTOR UNPACK extends half the elements of the source vector.
     case Intrinsic::s390_vuphb:  // VECTOR UNPACK HIGH
     case Intrinsic::s390_vuphh:
     case Intrinsic::s390_vuphf:
     case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
     case Intrinsic::s390_vuplhh:
     case Intrinsic::s390_vuplhf:
       SrcDemE = APInt(NumElts * 2, 0);
       SrcDemE.insertBits(DemandedElts, 0);
       break;
     case Intrinsic::s390_vuplb:  // VECTOR UNPACK LOW
     case Intrinsic::s390_vuplhw:
     case Intrinsic::s390_vuplf:
     case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
     case Intrinsic::s390_vupllh:
     case Intrinsic::s390_vupllf:
       SrcDemE = APInt(NumElts * 2, 0);
       SrcDemE.insertBits(DemandedElts, NumElts);
       break;
     case Intrinsic::s390_vpdi: {
       // VECTOR PERMUTE DWORD IMMEDIATE selects one element from each source.
       SrcDemE = APInt(NumElts, 0);
       if (!DemandedElts[OpNo - 1])
         break;
       unsigned Mask = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
       unsigned MaskBit = ((OpNo - 1) ? 1 : 4);
       // Demand input element 0 or 1, given by the mask bit value.
       SrcDemE.setBit((Mask & MaskBit)? 1 : 0);
       break;
     }
     case Intrinsic::s390_vsldb: {
       // VECTOR SHIFT LEFT DOUBLE BY BYTE
       assert(VT == MVT::v16i8 && "Unexpected type.");
       unsigned FirstIdx = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
       assert (FirstIdx > 0 && FirstIdx < 16 && "Unused operand.");
       unsigned NumSrc0Els = 16 - FirstIdx;
       SrcDemE = APInt(NumElts, 0);
       if (OpNo == 1) {
         APInt DemEls = DemandedElts.trunc(NumSrc0Els);
         SrcDemE.insertBits(DemEls, FirstIdx);
       } else {
         APInt DemEls = DemandedElts.lshr(NumSrc0Els);
         SrcDemE.insertBits(DemEls, 0);
       }
       break;
     }
     case Intrinsic::s390_vperm:
       SrcDemE = APInt(NumElts, 1);
       break;
     default:
       llvm_unreachable("Unhandled intrinsic.");
       break;
     }
   } else {
     switch (Opcode) {
     case SystemZISD::JOIN_DWORDS:
       // Scalar operand.
       SrcDemE = APInt(1, 1);
       break;
     case SystemZISD::SELECT_CCMASK:
       SrcDemE = DemandedElts;
       break;
     default:
       llvm_unreachable("Unhandled opcode.");
       break;
     }
   }
   return SrcDemE;
 }
 
 static void computeKnownBitsBinOp(const SDValue Op, KnownBits &Known,
                                   const APInt &DemandedElts,
                                   const SelectionDAG &DAG, unsigned Depth,
                                   unsigned OpNo) {
   APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
   APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo + 1);
   KnownBits LHSKnown =
       DAG.computeKnownBits(Op.getOperand(OpNo), Src0DemE, Depth + 1);
   KnownBits RHSKnown =
       DAG.computeKnownBits(Op.getOperand(OpNo + 1), Src1DemE, Depth + 1);
   Known = KnownBits::commonBits(LHSKnown, RHSKnown);
 }
 
 void
 SystemZTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
                                                      KnownBits &Known,
                                                      const APInt &DemandedElts,
                                                      const SelectionDAG &DAG,
                                                      unsigned Depth) const {
   Known.resetAll();
 
   // Intrinsic CC result is returned in the two low bits.
   unsigned tmp0, tmp1; // not used
   if (Op.getResNo() == 1 && isIntrinsicWithCC(Op, tmp0, tmp1)) {
     Known.Zero.setBitsFrom(2);
     return;
   }
   EVT VT = Op.getValueType();
   if (Op.getResNo() != 0 || VT == MVT::Untyped)
     return;
   assert (Known.getBitWidth() == VT.getScalarSizeInBits() &&
           "KnownBits does not match VT in bitwidth");
   assert ((!VT.isVector() ||
            (DemandedElts.getBitWidth() == VT.getVectorNumElements())) &&
           "DemandedElts does not match VT number of elements");
   unsigned BitWidth = Known.getBitWidth();
   unsigned Opcode = Op.getOpcode();
   if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
     bool IsLogical = false;
     unsigned Id = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
     switch (Id) {
     case Intrinsic::s390_vpksh:   // PACKS
     case Intrinsic::s390_vpksf:
     case Intrinsic::s390_vpksg:
     case Intrinsic::s390_vpkshs:  // PACKS_CC
     case Intrinsic::s390_vpksfs:
     case Intrinsic::s390_vpksgs:
     case Intrinsic::s390_vpklsh:  // PACKLS
     case Intrinsic::s390_vpklsf:
     case Intrinsic::s390_vpklsg:
     case Intrinsic::s390_vpklshs: // PACKLS_CC
     case Intrinsic::s390_vpklsfs:
     case Intrinsic::s390_vpklsgs:
     case Intrinsic::s390_vpdi:
     case Intrinsic::s390_vsldb:
     case Intrinsic::s390_vperm:
       computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, 1);
       break;
     case Intrinsic::s390_vuplhb: // VECTOR UNPACK LOGICAL HIGH
     case Intrinsic::s390_vuplhh:
     case Intrinsic::s390_vuplhf:
     case Intrinsic::s390_vupllb: // VECTOR UNPACK LOGICAL LOW
     case Intrinsic::s390_vupllh:
     case Intrinsic::s390_vupllf:
       IsLogical = true;
       LLVM_FALLTHROUGH;
     case Intrinsic::s390_vuphb:  // VECTOR UNPACK HIGH
     case Intrinsic::s390_vuphh:
     case Intrinsic::s390_vuphf:
     case Intrinsic::s390_vuplb:  // VECTOR UNPACK LOW
     case Intrinsic::s390_vuplhw:
     case Intrinsic::s390_vuplf: {
       SDValue SrcOp = Op.getOperand(1);
       APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, 0);
       Known = DAG.computeKnownBits(SrcOp, SrcDemE, Depth + 1);
       if (IsLogical) {
         Known = Known.zext(BitWidth);
       } else
         Known = Known.sext(BitWidth);
       break;
     }
     default:
       break;
     }
   } else {
     switch (Opcode) {
     case SystemZISD::JOIN_DWORDS:
     case SystemZISD::SELECT_CCMASK:
       computeKnownBitsBinOp(Op, Known, DemandedElts, DAG, Depth, 0);
       break;
     case SystemZISD::REPLICATE: {
       SDValue SrcOp = Op.getOperand(0);
       Known = DAG.computeKnownBits(SrcOp, Depth + 1);
       if (Known.getBitWidth() < BitWidth && isa<ConstantSDNode>(SrcOp))
         Known = Known.sext(BitWidth); // VREPI sign extends the immedate.
       break;
     }
     default:
       break;
     }
   }
 
   // Known has the width of the source operand(s). Adjust if needed to match
   // the passed bitwidth.
   if (Known.getBitWidth() != BitWidth)
     Known = Known.anyextOrTrunc(BitWidth);
 }
 
 static unsigned computeNumSignBitsBinOp(SDValue Op, const APInt &DemandedElts,
                                         const SelectionDAG &DAG, unsigned Depth,
                                         unsigned OpNo) {
   APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
   unsigned LHS = DAG.ComputeNumSignBits(Op.getOperand(OpNo), Src0DemE, Depth + 1);
   if (LHS == 1) return 1; // Early out.
   APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo + 1);
   unsigned RHS = DAG.ComputeNumSignBits(Op.getOperand(OpNo + 1), Src1DemE, Depth + 1);
   if (RHS == 1) return 1; // Early out.
   unsigned Common = std::min(LHS, RHS);
   unsigned SrcBitWidth = Op.getOperand(OpNo).getScalarValueSizeInBits();
   EVT VT = Op.getValueType();
   unsigned VTBits = VT.getScalarSizeInBits();
   if (SrcBitWidth > VTBits) { // PACK
     unsigned SrcExtraBits = SrcBitWidth - VTBits;
     if (Common > SrcExtraBits)
       return (Common - SrcExtraBits);
     return 1;
   }
   assert (SrcBitWidth == VTBits && "Expected operands of same bitwidth.");
   return Common;
 }
 
 unsigned
 SystemZTargetLowering::ComputeNumSignBitsForTargetNode(
     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
     unsigned Depth) const {
   if (Op.getResNo() != 0)
     return 1;
   unsigned Opcode = Op.getOpcode();
   if (Opcode == ISD::INTRINSIC_WO_CHAIN) {
     unsigned Id = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
     switch (Id) {
     case Intrinsic::s390_vpksh:   // PACKS
     case Intrinsic::s390_vpksf:
     case Intrinsic::s390_vpksg:
     case Intrinsic::s390_vpkshs:  // PACKS_CC
     case Intrinsic::s390_vpksfs:
     case Intrinsic::s390_vpksgs:
     case Intrinsic::s390_vpklsh:  // PACKLS
     case Intrinsic::s390_vpklsf:
     case Intrinsic::s390_vpklsg:
     case Intrinsic::s390_vpklshs: // PACKLS_CC
     case Intrinsic::s390_vpklsfs:
     case Intrinsic::s390_vpklsgs:
     case Intrinsic::s390_vpdi:
     case Intrinsic::s390_vsldb:
     case Intrinsic::s390_vperm:
       return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, 1);
     case Intrinsic::s390_vuphb:  // VECTOR UNPACK HIGH
     case Intrinsic::s390_vuphh:
     case Intrinsic::s390_vuphf:
     case Intrinsic::s390_vuplb:  // VECTOR UNPACK LOW
     case Intrinsic::s390_vuplhw:
     case Intrinsic::s390_vuplf: {
       SDValue PackedOp = Op.getOperand(1);
       APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, 1);
       unsigned Tmp = DAG.ComputeNumSignBits(PackedOp, SrcDemE, Depth + 1);
       EVT VT = Op.getValueType();
       unsigned VTBits = VT.getScalarSizeInBits();
       Tmp += VTBits - PackedOp.getScalarValueSizeInBits();
       return Tmp;
     }
     default:
       break;
     }
   } else {
     switch (Opcode) {
     case SystemZISD::SELECT_CCMASK:
       return computeNumSignBitsBinOp(Op, DemandedElts, DAG, Depth, 0);
     default:
       break;
     }
   }
 
   return 1;
 }
 
 unsigned
 SystemZTargetLowering::getStackProbeSize(MachineFunction &MF) const {
   const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
   unsigned StackAlign = TFI->getStackAlignment();
   assert(StackAlign >=1 && isPowerOf2_32(StackAlign) &&
          "Unexpected stack alignment");
   // The default stack probe size is 4096 if the function has no
   // stack-probe-size attribute.
   unsigned StackProbeSize = 4096;
   const Function &Fn = MF.getFunction();
   if (Fn.hasFnAttribute("stack-probe-size"))
     Fn.getFnAttribute("stack-probe-size")
         .getValueAsString()
         .getAsInteger(0, StackProbeSize);
   // Round down to the stack alignment.
   StackProbeSize &= ~(StackAlign - 1);
   return StackProbeSize ? StackProbeSize : StackAlign;
 }
 
 //===----------------------------------------------------------------------===//
 // Custom insertion
 //===----------------------------------------------------------------------===//
 
 // Force base value Base into a register before MI.  Return the register.
 static Register forceReg(MachineInstr &MI, MachineOperand &Base,
                          const SystemZInstrInfo *TII) {
   MachineBasicBlock *MBB = MI.getParent();
   MachineFunction &MF = *MBB->getParent();
   MachineRegisterInfo &MRI = MF.getRegInfo();
 
   if (Base.isReg()) {
     // Copy Base into a new virtual register to help register coalescing in
     // cases with multiple uses.
     Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
     BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::COPY), Reg)
       .add(Base);
     return Reg;
   }
 
   Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
   BuildMI(*MBB, MI, MI.getDebugLoc(), TII->get(SystemZ::LA), Reg)
       .add(Base)
       .addImm(0)
       .addReg(0);
   return Reg;
 }
 
 // The CC operand of MI might be missing a kill marker because there
 // were multiple uses of CC, and ISel didn't know which to mark.
 // Figure out whether MI should have had a kill marker.
 static bool checkCCKill(MachineInstr &MI, MachineBasicBlock *MBB) {
   // Scan forward through BB for a use/def of CC.
   MachineBasicBlock::iterator miI(std::next(MachineBasicBlock::iterator(MI)));
   for (MachineBasicBlock::iterator miE = MBB->end(); miI != miE; ++miI) {
     const MachineInstr& mi = *miI;
     if (mi.readsRegister(SystemZ::CC))
       return false;
     if (mi.definesRegister(SystemZ::CC))
       break; // Should have kill-flag - update below.
   }
 
   // If we hit the end of the block, check whether CC is live into a
   // successor.
   if (miI == MBB->end()) {
     for (const MachineBasicBlock *Succ : MBB->successors())
       if (Succ->isLiveIn(SystemZ::CC))
         return false;
   }
 
   return true;
 }
 
 // Return true if it is OK for this Select pseudo-opcode to be cascaded
 // together with other Select pseudo-opcodes into a single basic-block with
 // a conditional jump around it.
 static bool isSelectPseudo(MachineInstr &MI) {
   switch (MI.getOpcode()) {
   case SystemZ::Select32:
   case SystemZ::Select64:
   case SystemZ::SelectF32:
   case SystemZ::SelectF64:
   case SystemZ::SelectF128:
   case SystemZ::SelectVR32:
   case SystemZ::SelectVR64:
   case SystemZ::SelectVR128:
     return true;
 
   default:
     return false;
   }
 }
 
 // Helper function, which inserts PHI functions into SinkMBB:
 //   %Result(i) = phi [ %FalseValue(i), FalseMBB ], [ %TrueValue(i), TrueMBB ],
 // where %FalseValue(i) and %TrueValue(i) are taken from Selects.
 static void createPHIsForSelects(SmallVector<MachineInstr*, 8> &Selects,
                                  MachineBasicBlock *TrueMBB,
                                  MachineBasicBlock *FalseMBB,
                                  MachineBasicBlock *SinkMBB) {
   MachineFunction *MF = TrueMBB->getParent();
   const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
 
   MachineInstr *FirstMI = Selects.front();
   unsigned CCValid = FirstMI->getOperand(3).getImm();
   unsigned CCMask = FirstMI->getOperand(4).getImm();
 
   MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
 
   // As we are creating the PHIs, we have to be careful if there is more than
   // one.  Later Selects may reference the results of earlier Selects, but later
   // PHIs have to reference the individual true/false inputs from earlier PHIs.
   // That also means that PHI construction must work forward from earlier to
   // later, and that the code must maintain a mapping from earlier PHI's
   // destination registers, and the registers that went into the PHI.
   DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
 
   for (auto MI : Selects) {
     Register DestReg = MI->getOperand(0).getReg();
     Register TrueReg = MI->getOperand(1).getReg();
     Register FalseReg = MI->getOperand(2).getReg();
 
     // If this Select we are generating is the opposite condition from
     // the jump we generated, then we have to swap the operands for the
     // PHI that is going to be generated.
     if (MI->getOperand(4).getImm() == (CCValid ^ CCMask))
       std::swap(TrueReg, FalseReg);
 
     if (RegRewriteTable.find(TrueReg) != RegRewriteTable.end())
       TrueReg = RegRewriteTable[TrueReg].first;
 
     if (RegRewriteTable.find(FalseReg) != RegRewriteTable.end())
       FalseReg = RegRewriteTable[FalseReg].second;
 
     DebugLoc DL = MI->getDebugLoc();
     BuildMI(*SinkMBB, SinkInsertionPoint, DL, TII->get(SystemZ::PHI), DestReg)
       .addReg(TrueReg).addMBB(TrueMBB)
       .addReg(FalseReg).addMBB(FalseMBB);
 
     // Add this PHI to the rewrite table.
     RegRewriteTable[DestReg] = std::make_pair(TrueReg, FalseReg);
   }
 
   MF->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
 }
 
 // Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI.
 MachineBasicBlock *
 SystemZTargetLowering::emitSelect(MachineInstr &MI,
                                   MachineBasicBlock *MBB) const {
   assert(isSelectPseudo(MI) && "Bad call to emitSelect()");
   const SystemZInstrInfo *TII =
       static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
 
   unsigned CCValid = MI.getOperand(3).getImm();
   unsigned CCMask = MI.getOperand(4).getImm();
 
   // If we have a sequence of Select* pseudo instructions using the
   // same condition code value, we want to expand all of them into
   // a single pair of basic blocks using the same condition.
   SmallVector<MachineInstr*, 8> Selects;
   SmallVector<MachineInstr*, 8> DbgValues;
   Selects.push_back(&MI);
   unsigned Count = 0;
   for (MachineBasicBlock::iterator NextMIIt =
          std::next(MachineBasicBlock::iterator(MI));
        NextMIIt != MBB->end(); ++NextMIIt) {
     if (isSelectPseudo(*NextMIIt)) {
       assert(NextMIIt->getOperand(3).getImm() == CCValid &&
              "Bad CCValid operands since CC was not redefined.");
       if (NextMIIt->getOperand(4).getImm() == CCMask ||
           NextMIIt->getOperand(4).getImm() == (CCValid ^ CCMask)) {
         Selects.push_back(&*NextMIIt);
         continue;
       }
       break;
     }
     if (NextMIIt->definesRegister(SystemZ::CC) ||
         NextMIIt->usesCustomInsertionHook())
       break;
     bool User = false;
     for (auto SelMI : Selects)
       if (NextMIIt->readsVirtualRegister(SelMI->getOperand(0).getReg())) {
         User = true;
         break;
       }
     if (NextMIIt->isDebugInstr()) {
       if (User) {
         assert(NextMIIt->isDebugValue() && "Unhandled debug opcode.");
         DbgValues.push_back(&*NextMIIt);
       }
     }
     else if (User || ++Count > 20)
       break;
   }
 
   MachineInstr *LastMI = Selects.back();
   bool CCKilled =
       (LastMI->killsRegister(SystemZ::CC) || checkCCKill(*LastMI, MBB));
   MachineBasicBlock *StartMBB = MBB;
   MachineBasicBlock *JoinMBB  = SystemZ::splitBlockAfter(LastMI, MBB);
   MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(StartMBB);
 
   // Unless CC was killed in the last Select instruction, mark it as
   // live-in to both FalseMBB and JoinMBB.
   if (!CCKilled) {
     FalseMBB->addLiveIn(SystemZ::CC);
     JoinMBB->addLiveIn(SystemZ::CC);
   }
 
   //  StartMBB:
   //   BRC CCMask, JoinMBB
   //   # fallthrough to FalseMBB
   MBB = StartMBB;
   BuildMI(MBB, MI.getDebugLoc(), TII->get(SystemZ::BRC))
     .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB);
   MBB->addSuccessor(JoinMBB);
   MBB->addSuccessor(FalseMBB);
 
   //  FalseMBB:
   //   # fallthrough to JoinMBB
   MBB = FalseMBB;
   MBB->addSuccessor(JoinMBB);
 
   //  JoinMBB:
   //   %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ]
   //  ...
   MBB = JoinMBB;
   createPHIsForSelects(Selects, StartMBB, FalseMBB, MBB);
   for (auto SelMI : Selects)
     SelMI->eraseFromParent();
 
   MachineBasicBlock::iterator InsertPos = MBB->getFirstNonPHI();
   for (auto DbgMI : DbgValues)
     MBB->splice(InsertPos, StartMBB, DbgMI);
 
   return JoinMBB;
 }
 
 // Implement EmitInstrWithCustomInserter for pseudo CondStore* instruction MI.
 // StoreOpcode is the store to use and Invert says whether the store should
 // happen when the condition is false rather than true.  If a STORE ON
 // CONDITION is available, STOCOpcode is its opcode, otherwise it is 0.
 MachineBasicBlock *SystemZTargetLowering::emitCondStore(MachineInstr &MI,
                                                         MachineBasicBlock *MBB,
                                                         unsigned StoreOpcode,
                                                         unsigned STOCOpcode,
                                                         bool Invert) const {
   const SystemZInstrInfo *TII =
       static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
 
   Register SrcReg = MI.getOperand(0).getReg();
   MachineOperand Base = MI.getOperand(1);
   int64_t Disp = MI.getOperand(2).getImm();
   Register IndexReg = MI.getOperand(3).getReg();
   unsigned CCValid = MI.getOperand(4).getImm();
   unsigned CCMask = MI.getOperand(5).getImm();
   DebugLoc DL = MI.getDebugLoc();
 
   StoreOpcode = TII->getOpcodeForOffset(StoreOpcode, Disp);
 
   // ISel pattern matching also adds a load memory operand of the same
   // address, so take special care to find the storing memory operand.
   MachineMemOperand *MMO = nullptr;
   for (auto *I : MI.memoperands())
     if (I->isStore()) {
       MMO = I;
       break;
     }
 
   // Use STOCOpcode if possible.  We could use different store patterns in
   // order to avoid matching the index register, but the performance trade-offs
   // might be more complicated in that case.
   if (STOCOpcode && !IndexReg && Subtarget.hasLoadStoreOnCond()) {
     if (Invert)
       CCMask ^= CCValid;
 
     BuildMI(*MBB, MI, DL, TII->get(STOCOpcode))
       .addReg(SrcReg)
       .add(Base)
       .addImm(Disp)
       .addImm(CCValid)
       .addImm(CCMask)
       .addMemOperand(MMO);
 
     MI.eraseFromParent();
     return MBB;
   }
 
   // Get the condition needed to branch around the store.
   if (!Invert)
     CCMask ^= CCValid;
 
   MachineBasicBlock *StartMBB = MBB;
   MachineBasicBlock *JoinMBB  = SystemZ::splitBlockBefore(MI, MBB);
   MachineBasicBlock *FalseMBB = SystemZ::emitBlockAfter(StartMBB);
 
   // Unless CC was killed in the CondStore instruction, mark it as
   // live-in to both FalseMBB and JoinMBB.
   if (!MI.killsRegister(SystemZ::CC) && !checkCCKill(MI, JoinMBB)) {
     FalseMBB->addLiveIn(SystemZ::CC);
     JoinMBB->addLiveIn(SystemZ::CC);
   }
 
   //  StartMBB:
   //   BRC CCMask, JoinMBB
   //   # fallthrough to FalseMBB
   MBB = StartMBB;
   BuildMI(MBB, DL, TII->get(SystemZ::BRC))
     .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB);
   MBB->addSuccessor(JoinMBB);
   MBB->addSuccessor(FalseMBB);
 
   //  FalseMBB:
   //   store %SrcReg, %Disp(%Index,%Base)
   //   # fallthrough to JoinMBB
   MBB = FalseMBB;
   BuildMI(MBB, DL, TII->get(StoreOpcode))
       .addReg(SrcReg)
       .add(Base)
       .addImm(Disp)
       .addReg(IndexReg)
       .addMemOperand(MMO);
   MBB->addSuccessor(JoinMBB);
 
   MI.eraseFromParent();
   return JoinMBB;
 }
 
 // Implement EmitInstrWithCustomInserter for pseudo ATOMIC_LOAD{,W}_*
 // or ATOMIC_SWAP{,W} instruction MI.  BinOpcode is the instruction that
 // performs the binary operation elided by "*", or 0 for ATOMIC_SWAP{,W}.
 // BitSize is the width of the field in bits, or 0 if this is a partword
 // ATOMIC_LOADW_* or ATOMIC_SWAPW instruction, in which case the bitsize
 // is one of the operands.  Invert says whether the field should be
 // inverted after performing BinOpcode (e.g. for NAND).
 MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary(
     MachineInstr &MI, MachineBasicBlock *MBB, unsigned BinOpcode,
     unsigned BitSize, bool Invert) const {
   MachineFunction &MF = *MBB->getParent();
   const SystemZInstrInfo *TII =
       static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
   MachineRegisterInfo &MRI = MF.getRegInfo();
   bool IsSubWord = (BitSize < 32);
 
   // Extract the operands.  Base can be a register or a frame index.
   // Src2 can be a register or immediate.
   Register Dest = MI.getOperand(0).getReg();
   MachineOperand Base = earlyUseOperand(MI.getOperand(1));
   int64_t Disp = MI.getOperand(2).getImm();
   MachineOperand Src2 = earlyUseOperand(MI.getOperand(3));
   Register BitShift = IsSubWord ? MI.getOperand(4).getReg() : Register();
   Register NegBitShift = IsSubWord ? MI.getOperand(5).getReg() : Register();
   DebugLoc DL = MI.getDebugLoc();
   if (IsSubWord)
     BitSize = MI.getOperand(6).getImm();
 
   // Subword operations use 32-bit registers.
   const TargetRegisterClass *RC = (BitSize <= 32 ?
                                    &SystemZ::GR32BitRegClass :
                                    &SystemZ::GR64BitRegClass);
   unsigned LOpcode  = BitSize <= 32 ? SystemZ::L  : SystemZ::LG;
   unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG;
 
   // Get the right opcodes for the displacement.
   LOpcode  = TII->getOpcodeForOffset(LOpcode,  Disp);
   CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp);
   assert(LOpcode && CSOpcode && "Displacement out of range");
 
   // Create virtual registers for temporary results.
   Register OrigVal       = MRI.createVirtualRegister(RC);
   Register OldVal        = MRI.createVirtualRegister(RC);
   Register NewVal        = (BinOpcode || IsSubWord ?
                             MRI.createVirtualRegister(RC) : Src2.getReg());
   Register RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal);
   Register RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal);
 
   // Insert a basic block for the main loop.
   MachineBasicBlock *StartMBB = MBB;
   MachineBasicBlock *DoneMBB  = SystemZ::splitBlockBefore(MI, MBB);
   MachineBasicBlock *LoopMBB  = SystemZ::emitBlockAfter(StartMBB);
 
   //  StartMBB:
   //   ...
   //   %OrigVal = L Disp(%Base)
   //   # fall through to LoopMBB
   MBB = StartMBB;
   BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0);
   MBB->addSuccessor(LoopMBB);
 
   //  LoopMBB:
   //   %OldVal        = phi [ %OrigVal, StartMBB ], [ %Dest, LoopMBB ]
   //   %RotatedOldVal = RLL %OldVal, 0(%BitShift)
   //   %RotatedNewVal = OP %RotatedOldVal, %Src2
   //   %NewVal        = RLL %RotatedNewVal, 0(%NegBitShift)
   //   %Dest          = CS %OldVal, %NewVal, Disp(%Base)
   //   JNE LoopMBB
   //   # fall through to DoneMBB
   MBB = LoopMBB;
   BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
     .addReg(OrigVal).addMBB(StartMBB)
     .addReg(Dest).addMBB(LoopMBB);
   if (IsSubWord)
     BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal)
       .addReg(OldVal).addReg(BitShift).addImm(0);
   if (Invert) {
     // Perform the operation normally and then invert every bit of the field.
     Register Tmp = MRI.createVirtualRegister(RC);
     BuildMI(MBB, DL, TII->get(BinOpcode), Tmp).addReg(RotatedOldVal).add(Src2);
     if (BitSize <= 32)
       // XILF with the upper BitSize bits set.
       BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal)
         .addReg(Tmp).addImm(-1U << (32 - BitSize));
     else {
       // Use LCGR and add -1 to the result, which is more compact than
       // an XILF, XILH pair.
       Register Tmp2 = MRI.createVirtualRegister(RC);
       BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp);
       BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal)
         .addReg(Tmp2).addImm(-1);
     }
   } else if (BinOpcode)
     // A simply binary operation.
     BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal)
         .addReg(RotatedOldVal)
         .add(Src2);
   else if (IsSubWord)
     // Use RISBG to rotate Src2 into position and use it to replace the
     // field in RotatedOldVal.
     BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal)
       .addReg(RotatedOldVal).addReg(Src2.getReg())
       .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize);
   if (IsSubWord)
     BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal)
       .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0);
   BuildMI(MBB, DL, TII->get(CSOpcode), Dest)
       .addReg(OldVal)
       .addReg(NewVal)
       .add(Base)
       .addImm(Disp);
   BuildMI(MBB, DL, TII->get(SystemZ::BRC))
     .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB);
   MBB->addSuccessor(LoopMBB);
   MBB->addSuccessor(DoneMBB);
 
   MI.eraseFromParent();
   return DoneMBB;
 }
 
 // Implement EmitInstrWithCustomInserter for pseudo
 // ATOMIC_LOAD{,W}_{,U}{MIN,MAX} instruction MI.  CompareOpcode is the
 // instruction that should be used to compare the current field with the
 // minimum or maximum value.  KeepOldMask is the BRC condition-code mask
 // for when the current field should be kept.  BitSize is the width of
 // the field in bits, or 0 if this is a partword ATOMIC_LOADW_* instruction.
 MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
     MachineInstr &MI, MachineBasicBlock *MBB, unsigned CompareOpcode,
     unsigned KeepOldMask, unsigned BitSize) const {
   MachineFunction &MF = *MBB->getParent();
   const SystemZInstrInfo *TII =
       static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
   MachineRegisterInfo &MRI = MF.getRegInfo();
   bool IsSubWord = (BitSize < 32);
 
   // Extract the operands.  Base can be a register or a frame index.
   Register Dest = MI.getOperand(0).getReg();
   MachineOperand Base = earlyUseOperand(MI.getOperand(1));
   int64_t Disp = MI.getOperand(2).getImm();
   Register Src2 = MI.getOperand(3).getReg();
   Register BitShift = (IsSubWord ? MI.getOperand(4).getReg() : Register());
   Register NegBitShift = (IsSubWord ? MI.getOperand(5).getReg() : Register());
   DebugLoc DL = MI.getDebugLoc();
   if (IsSubWord)
     BitSize = MI.getOperand(6).getImm();
 
   // Subword operations use 32-bit registers.
   const TargetRegisterClass *RC = (BitSize <= 32 ?
                                    &SystemZ::GR32BitRegClass :
                                    &SystemZ::GR64BitRegClass);
   unsigned LOpcode  = BitSize <= 32 ? SystemZ::L  : SystemZ::LG;
   unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG;
 
   // Get the right opcodes for the displacement.
   LOpcode  = TII->getOpcodeForOffset(LOpcode,  Disp);
   CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp);
   assert(LOpcode && CSOpcode && "Displacement out of range");
 
   // Create virtual registers for temporary results.
   Register OrigVal       = MRI.createVirtualRegister(RC);
   Register OldVal        = MRI.createVirtualRegister(RC);
   Register NewVal        = MRI.createVirtualRegister(RC);
   Register RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal);
   Register RotatedAltVal = (IsSubWord ? MRI.createVirtualRegister(RC) : Src2);
   Register RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal);
 
   // Insert 3 basic blocks for the loop.
   MachineBasicBlock *StartMBB  = MBB;
   MachineBasicBlock *DoneMBB   = SystemZ::splitBlockBefore(MI, MBB);
   MachineBasicBlock *LoopMBB   = SystemZ::emitBlockAfter(StartMBB);
   MachineBasicBlock *UseAltMBB = SystemZ::emitBlockAfter(LoopMBB);
   MachineBasicBlock *UpdateMBB = SystemZ::emitBlockAfter(UseAltMBB);
 
   //  StartMBB:
   //   ...
   //   %OrigVal     = L Disp(%Base)
   //   # fall through to LoopMBB
   MBB = StartMBB;
   BuildMI(MBB, DL, TII->get(LOpcode), OrigVal).add(Base).addImm(Disp).addReg(0);
   MBB->addSuccessor(LoopMBB);
 
   //  LoopMBB:
   //   %OldVal        = phi [ %OrigVal, StartMBB ], [ %Dest, UpdateMBB ]
   //   %RotatedOldVal = RLL %OldVal, 0(%BitShift)
   //   CompareOpcode %RotatedOldVal, %Src2
   //   BRC KeepOldMask, UpdateMBB
   MBB = LoopMBB;
   BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
     .addReg(OrigVal).addMBB(StartMBB)
     .addReg(Dest).addMBB(UpdateMBB);
   if (IsSubWord)
     BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal)
       .addReg(OldVal).addReg(BitShift).addImm(0);
   BuildMI(MBB, DL, TII->get(CompareOpcode))
     .addReg(RotatedOldVal).addReg(Src2);
   BuildMI(MBB, DL, TII->get(SystemZ::BRC))
     .addImm(SystemZ::CCMASK_ICMP).addImm(KeepOldMask).addMBB(UpdateMBB);
   MBB->addSuccessor(UpdateMBB);
   MBB->addSuccessor(UseAltMBB);
 
   //  UseAltMBB:
   //   %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0
   //   # fall through to UpdateMBB
   MBB = UseAltMBB;
   if (IsSubWord)
     BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal)
       .addReg(RotatedOldVal).addReg(Src2)
       .addImm(32).addImm(31 + BitSize).addImm(0);
   MBB->addSuccessor(UpdateMBB);
 
   //  UpdateMBB:
   //   %RotatedNewVal = PHI [ %RotatedOldVal, LoopMBB ],
   //                        [ %RotatedAltVal, UseAltMBB ]
   //   %NewVal        = RLL %RotatedNewVal, 0(%NegBitShift)
   //   %Dest          = CS %OldVal, %NewVal, Disp(%Base)
   //   JNE LoopMBB
   //   # fall through to DoneMBB
   MBB = UpdateMBB;
   BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal)
     .addReg(RotatedOldVal).addMBB(LoopMBB)
     .addReg(RotatedAltVal).addMBB(UseAltMBB);
   if (IsSubWord)
     BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal)
       .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0);
   BuildMI(MBB, DL, TII->get(CSOpcode), Dest)
       .addReg(OldVal)
       .addReg(NewVal)
       .add(Base)
       .addImm(Disp);
   BuildMI(MBB, DL, TII->get(SystemZ::BRC))
     .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB);
   MBB->addSuccessor(LoopMBB);
   MBB->addSuccessor(DoneMBB);
 
   MI.eraseFromParent();
   return DoneMBB;
 }
 
 // Implement EmitInstrWithCustomInserter for pseudo ATOMIC_CMP_SWAPW
 // instruction MI.
 MachineBasicBlock *
 SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
                                           MachineBasicBlock *MBB) const {
   MachineFunction &MF = *MBB->getParent();
   const SystemZInstrInfo *TII =
       static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
   MachineRegisterInfo &MRI = MF.getRegInfo();
 
   // Extract the operands.  Base can be a register or a frame index.
   Register Dest = MI.getOperand(0).getReg();
   MachineOperand Base = earlyUseOperand(MI.getOperand(1));
   int64_t Disp = MI.getOperand(2).getImm();
   Register CmpVal = MI.getOperand(3).getReg();
   Register OrigSwapVal = MI.getOperand(4).getReg();
   Register BitShift = MI.getOperand(5).getReg();
   Register NegBitShift = MI.getOperand(6).getReg();
   int64_t BitSize = MI.getOperand(7).getImm();
   DebugLoc DL = MI.getDebugLoc();
 
   const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass;
 
   // Get the right opcodes for the displacement and zero-extension.
   unsigned LOpcode  = TII->getOpcodeForOffset(SystemZ::L,  Disp);
   unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);
   unsigned ZExtOpcode  = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR;
   assert(LOpcode && CSOpcode && "Displacement out of range");
 
   // Create virtual registers for temporary results.
   Register OrigOldVal = MRI.createVirtualRegister(RC);
   Register OldVal = MRI.createVirtualRegister(RC);
   Register SwapVal = MRI.createVirtualRegister(RC);
   Register StoreVal = MRI.createVirtualRegister(RC);
   Register OldValRot = MRI.createVirtualRegister(RC);
   Register RetryOldVal = MRI.createVirtualRegister(RC);
   Register RetrySwapVal = MRI.createVirtualRegister(RC);
 
   // Insert 2 basic blocks for the loop.
   MachineBasicBlock *StartMBB = MBB;
   MachineBasicBlock *DoneMBB  = SystemZ::splitBlockBefore(MI, MBB);
   MachineBasicBlock *LoopMBB  = SystemZ::emitBlockAfter(StartMBB);
   MachineBasicBlock *SetMBB   = SystemZ::emitBlockAfter(LoopMBB);
 
   //  StartMBB:
   //   ...
   //   %OrigOldVal     = L Disp(%Base)
   //   # fall through to LoopMBB
   MBB = StartMBB;
   BuildMI(MBB, DL, TII->get(LOpcode), OrigOldVal)
       .add(Base)
       .addImm(Disp)
       .addReg(0);
   MBB->addSuccessor(LoopMBB);
 
   //  LoopMBB:
   //   %OldVal        = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ]
   //   %SwapVal       = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ]
   //   %OldValRot     = RLL %OldVal, BitSize(%BitShift)
   //                      ^^ The low BitSize bits contain the field
   //                         of interest.
   //   %RetrySwapVal = RISBG32 %SwapVal, %OldValRot, 32, 63-BitSize, 0
   //                      ^^ Replace the upper 32-BitSize bits of the
   //                         swap value with those that we loaded and rotated.
   //   %Dest = LL[CH] %OldValRot
   //   CR %Dest, %CmpVal
   //   JNE DoneMBB
   //   # Fall through to SetMBB
   MBB = LoopMBB;
   BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
     .addReg(OrigOldVal).addMBB(StartMBB)
     .addReg(RetryOldVal).addMBB(SetMBB);
   BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal)
     .addReg(OrigSwapVal).addMBB(StartMBB)
     .addReg(RetrySwapVal).addMBB(SetMBB);
   BuildMI(MBB, DL, TII->get(SystemZ::RLL), OldValRot)
     .addReg(OldVal).addReg(BitShift).addImm(BitSize);
   BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal)
     .addReg(SwapVal).addReg(OldValRot).addImm(32).addImm(63 - BitSize).addImm(0);
   BuildMI(MBB, DL, TII->get(ZExtOpcode), Dest)
     .addReg(OldValRot);
   BuildMI(MBB, DL, TII->get(SystemZ::CR))
     .addReg(Dest).addReg(CmpVal);
   BuildMI(MBB, DL, TII->get(SystemZ::BRC))
     .addImm(SystemZ::CCMASK_ICMP)
     .addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB);
   MBB->addSuccessor(DoneMBB);
   MBB->addSuccessor(SetMBB);
 
   //  SetMBB:
   //   %StoreVal     = RLL %RetrySwapVal, -BitSize(%NegBitShift)
   //                      ^^ Rotate the new field to its proper position.
   //   %RetryOldVal  = CS %OldVal, %StoreVal, Disp(%Base)
   //   JNE LoopMBB
   //   # fall through to ExitMBB
   MBB = SetMBB;
   BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal)
     .addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize);
   BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal)
       .addReg(OldVal)
       .addReg(StoreVal)
       .add(Base)
       .addImm(Disp);
   BuildMI(MBB, DL, TII->get(SystemZ::BRC))
     .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB);
   MBB->addSuccessor(LoopMBB);
   MBB->addSuccessor(DoneMBB);
 
   // If the CC def wasn't dead in the ATOMIC_CMP_SWAPW, mark CC as live-in
   // to the block after the loop.  At this point, CC may have been defined
   // either by the CR in LoopMBB or by the CS in SetMBB.
   if (!MI.registerDefIsDead(SystemZ::CC))
     DoneMBB->addLiveIn(SystemZ::CC);
 
   MI.eraseFromParent();
   return DoneMBB;
 }
 
 // Emit a move from two GR64s to a GR128.
 MachineBasicBlock *
 SystemZTargetLowering::emitPair128(MachineInstr &MI,
                                    MachineBasicBlock *MBB) const {
   MachineFunction &MF = *MBB->getParent();
   const SystemZInstrInfo *TII =
       static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
   MachineRegisterInfo &MRI = MF.getRegInfo();
   DebugLoc DL = MI.getDebugLoc();
 
   Register Dest = MI.getOperand(0).getReg();
   Register Hi = MI.getOperand(1).getReg();
   Register Lo = MI.getOperand(2).getReg();
   Register Tmp1 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
   Register Tmp2 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
 
   BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), Tmp1);
   BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Tmp2)
     .addReg(Tmp1).addReg(Hi).addImm(SystemZ::subreg_h64);
   BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest)
     .addReg(Tmp2).addReg(Lo).addImm(SystemZ::subreg_l64);
 
   MI.eraseFromParent();
   return MBB;
 }
 
 // Emit an extension from a GR64 to a GR128.  ClearEven is true
 // if the high register of the GR128 value must be cleared or false if
 // it's "don't care".
 MachineBasicBlock *SystemZTargetLowering::emitExt128(MachineInstr &MI,
                                                      MachineBasicBlock *MBB,
                                                      bool ClearEven) const {
   MachineFunction &MF = *MBB->getParent();
   const SystemZInstrInfo *TII =
       static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
   MachineRegisterInfo &MRI = MF.getRegInfo();
   DebugLoc DL = MI.getDebugLoc();
 
   Register Dest = MI.getOperand(0).getReg();
   Register Src = MI.getOperand(1).getReg();
   Register In128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
 
   BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), In128);
   if (ClearEven) {
     Register NewIn128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
     Register Zero64 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass);
 
     BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64)
       .addImm(0);
     BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewIn128)
       .addReg(In128).addReg(Zero64).addImm(SystemZ::subreg_h64);
     In128 = NewIn128;
   }
   BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dest)
     .addReg(In128).addReg(Src).addImm(SystemZ::subreg_l64);
 
   MI.eraseFromParent();
   return MBB;
 }
 
 MachineBasicBlock *
 SystemZTargetLowering::emitMemMemWrapper(MachineInstr &MI,
                                          MachineBasicBlock *MBB,
                                          unsigned Opcode, bool IsMemset) const {
   MachineFunction &MF = *MBB->getParent();
   const SystemZInstrInfo *TII =
       static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
   MachineRegisterInfo &MRI = MF.getRegInfo();
   DebugLoc DL = MI.getDebugLoc();
 
   MachineOperand DestBase = earlyUseOperand(MI.getOperand(0));
   uint64_t DestDisp = MI.getOperand(1).getImm();
   MachineOperand SrcBase = MachineOperand::CreateReg(0U, false);
   uint64_t SrcDisp;
 
   // Fold the displacement Disp if it is out of range.
   auto foldDisplIfNeeded = [&](MachineOperand &Base, uint64_t &Disp) -> void {
     if (!isUInt<12>(Disp)) {
       Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
       unsigned Opcode = TII->getOpcodeForOffset(SystemZ::LA, Disp);
       BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII->get(Opcode), Reg)
         .add(Base).addImm(Disp).addReg(0);
       Base = MachineOperand::CreateReg(Reg, false);
       Disp = 0;
     }
   };
 
   if (!IsMemset) {
     SrcBase = earlyUseOperand(MI.getOperand(2));
     SrcDisp = MI.getOperand(3).getImm();
   } else {
     SrcBase = DestBase;
     SrcDisp = DestDisp++;
     foldDisplIfNeeded(DestBase, DestDisp);
   }
 
   MachineOperand &LengthMO = MI.getOperand(IsMemset ? 2 : 4);
   bool IsImmForm = LengthMO.isImm();
   bool IsRegForm = !IsImmForm;
 
   // Build and insert one Opcode of Length, with special treatment for memset.
   auto insertMemMemOp = [&](MachineBasicBlock *InsMBB,
                             MachineBasicBlock::iterator InsPos,
                             MachineOperand DBase, uint64_t DDisp,
                             MachineOperand SBase, uint64_t SDisp,
                             unsigned Length) -> void {
     assert(Length > 0 && Length <= 256 && "Building memory op with bad length.");
     if (IsMemset) {
       MachineOperand ByteMO = earlyUseOperand(MI.getOperand(3));
       if (ByteMO.isImm())
         BuildMI(*InsMBB, InsPos, DL, TII->get(SystemZ::MVI))
           .add(SBase).addImm(SDisp).add(ByteMO);
       else
         BuildMI(*InsMBB, InsPos, DL, TII->get(SystemZ::STC))
           .add(ByteMO).add(SBase).addImm(SDisp).addReg(0);
       if (--Length == 0)
         return;
     }
     BuildMI(*MBB, InsPos, DL, TII->get(Opcode))
       .add(DBase).addImm(DDisp).addImm(Length)
       .add(SBase).addImm(SDisp)
       .setMemRefs(MI.memoperands());
   };
 
   bool NeedsLoop = false;
   uint64_t ImmLength = 0;
   Register LenAdjReg = SystemZ::NoRegister;
   if (IsImmForm) {
     ImmLength = LengthMO.getImm();
     ImmLength += IsMemset ? 2 : 1; // Add back the subtracted adjustment.
     if (ImmLength == 0) {
       MI.eraseFromParent();
       return MBB;
     }
     if (Opcode == SystemZ::CLC) {
       if (ImmLength > 3 * 256)
         // A two-CLC sequence is a clear win over a loop, not least because
         // it needs only one branch.  A three-CLC sequence needs the same
         // number of branches as a loop (i.e. 2), but is shorter.  That
         // brings us to lengths greater than 768 bytes.  It seems relatively
         // likely that a difference will be found within the first 768 bytes,
         // so we just optimize for the smallest number of branch
         // instructions, in order to avoid polluting the prediction buffer
         // too much.
         NeedsLoop = true;
     } else if (ImmLength > 6 * 256)
       // The heuristic we use is to prefer loops for anything that would
       // require 7 or more MVCs.  With these kinds of sizes there isn't much
       // to choose between straight-line code and looping code, since the
       // time will be dominated by the MVCs themselves.
       NeedsLoop = true;
   } else {
     NeedsLoop = true;
     LenAdjReg = LengthMO.getReg();
   }
 
   // When generating more than one CLC, all but the last will need to
   // branch to the end when a difference is found.
   MachineBasicBlock *EndMBB =
       (Opcode == SystemZ::CLC && (ImmLength > 256 || NeedsLoop)
            ? SystemZ::splitBlockAfter(MI, MBB)
            : nullptr);
 
   if (NeedsLoop) {
     Register StartCountReg =
       MRI.createVirtualRegister(&SystemZ::GR64BitRegClass);
     if (IsImmForm) {
       TII->loadImmediate(*MBB, MI, StartCountReg, ImmLength / 256);
       ImmLength &= 255;
     } else {
       BuildMI(*MBB, MI, DL, TII->get(SystemZ::SRLG), StartCountReg)
         .addReg(LenAdjReg)
         .addReg(0)
         .addImm(8);
     }
 
     bool HaveSingleBase = DestBase.isIdenticalTo(SrcBase);
     auto loadZeroAddress = [&]() -> MachineOperand {
       Register Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
       BuildMI(*MBB, MI, DL, TII->get(SystemZ::LGHI), Reg).addImm(0);
       return MachineOperand::CreateReg(Reg, false);
     };
     if (DestBase.isReg() && DestBase.getReg() == SystemZ::NoRegister)
       DestBase = loadZeroAddress();
     if (SrcBase.isReg() && SrcBase.getReg() == SystemZ::NoRegister)
       SrcBase = HaveSingleBase ? DestBase : loadZeroAddress();
 
     MachineBasicBlock *StartMBB = nullptr;
     MachineBasicBlock *LoopMBB = nullptr;
     MachineBasicBlock *NextMBB = nullptr;
     MachineBasicBlock *DoneMBB = nullptr;
     MachineBasicBlock *AllDoneMBB = nullptr;
 
     Register StartSrcReg = forceReg(MI, SrcBase, TII);
     Register StartDestReg =
         (HaveSingleBase ? StartSrcReg : forceReg(MI, DestBase, TII));
 
     const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
     Register ThisSrcReg  = MRI.createVirtualRegister(RC);
     Register ThisDestReg =
         (HaveSingleBase ? ThisSrcReg : MRI.createVirtualRegister(RC));
     Register NextSrcReg  = MRI.createVirtualRegister(RC);
     Register NextDestReg =
         (HaveSingleBase ? NextSrcReg : MRI.createVirtualRegister(RC));
     RC = &SystemZ::GR64BitRegClass;
     Register ThisCountReg = MRI.createVirtualRegister(RC);
     Register NextCountReg = MRI.createVirtualRegister(RC);
 
     if (IsRegForm) {
       AllDoneMBB = SystemZ::splitBlockBefore(MI, MBB);
       StartMBB = SystemZ::emitBlockAfter(MBB);
       LoopMBB = SystemZ::emitBlockAfter(StartMBB);
       NextMBB = (EndMBB ? SystemZ::emitBlockAfter(LoopMBB) : LoopMBB);
       DoneMBB = SystemZ::emitBlockAfter(NextMBB);
 
       //  MBB:
       //   # Jump to AllDoneMBB if LenAdjReg means 0, or fall thru to StartMBB.
       BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
         .addReg(LenAdjReg).addImm(IsMemset ? -2 : -1);
       BuildMI(MBB, DL, TII->get(SystemZ::BRC))
         .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_EQ)
         .addMBB(AllDoneMBB);
       MBB->addSuccessor(AllDoneMBB);
       if (!IsMemset)
         MBB->addSuccessor(StartMBB);
       else {
         // MemsetOneCheckMBB:
         // # Jump to MemsetOneMBB for a memset of length 1, or
         // # fall thru to StartMBB.
         MachineBasicBlock *MemsetOneCheckMBB = SystemZ::emitBlockAfter(MBB);
         MachineBasicBlock *MemsetOneMBB = SystemZ::emitBlockAfter(&*MF.rbegin());
         MBB->addSuccessor(MemsetOneCheckMBB);
         MBB = MemsetOneCheckMBB;
         BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
           .addReg(LenAdjReg).addImm(-1);
         BuildMI(MBB, DL, TII->get(SystemZ::BRC))
           .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_EQ)
           .addMBB(MemsetOneMBB);
         MBB->addSuccessor(MemsetOneMBB, {10, 100});
         MBB->addSuccessor(StartMBB, {90, 100});
 
         // MemsetOneMBB:
         // # Jump back to AllDoneMBB after a single MVI or STC.
         MBB = MemsetOneMBB;
         insertMemMemOp(MBB, MBB->end(),
                        MachineOperand::CreateReg(StartDestReg, false), DestDisp,
                        MachineOperand::CreateReg(StartSrcReg, false), SrcDisp,
                        1);
         BuildMI(MBB, DL, TII->get(SystemZ::J)).addMBB(AllDoneMBB);
         MBB->addSuccessor(AllDoneMBB);
       }
 
       // StartMBB:
       // # Jump to DoneMBB if %StartCountReg is zero, or fall through to LoopMBB.
       MBB = StartMBB;
       BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
         .addReg(StartCountReg).addImm(0);
       BuildMI(MBB, DL, TII->get(SystemZ::BRC))
         .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_EQ)
         .addMBB(DoneMBB);
       MBB->addSuccessor(DoneMBB);
       MBB->addSuccessor(LoopMBB);
     }
     else {
       StartMBB = MBB;
       DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
       LoopMBB = SystemZ::emitBlockAfter(StartMBB);
       NextMBB = (EndMBB ? SystemZ::emitBlockAfter(LoopMBB) : LoopMBB);
 
       //  StartMBB:
       //   # fall through to LoopMBB
       MBB->addSuccessor(LoopMBB);
 
       DestBase = MachineOperand::CreateReg(NextDestReg, false);
       SrcBase = MachineOperand::CreateReg(NextSrcReg, false);
       if (EndMBB && !ImmLength)
         // If the loop handled the whole CLC range, DoneMBB will be empty with
         // CC live-through into EndMBB, so add it as live-in.
         DoneMBB->addLiveIn(SystemZ::CC);
     }
 
     //  LoopMBB:
     //   %ThisDestReg = phi [ %StartDestReg, StartMBB ],
     //                      [ %NextDestReg, NextMBB ]
     //   %ThisSrcReg = phi [ %StartSrcReg, StartMBB ],
     //                     [ %NextSrcReg, NextMBB ]
     //   %ThisCountReg = phi [ %StartCountReg, StartMBB ],
     //                       [ %NextCountReg, NextMBB ]
     //   ( PFD 2, 768+DestDisp(%ThisDestReg) )
     //   Opcode DestDisp(256,%ThisDestReg), SrcDisp(%ThisSrcReg)
     //   ( JLH EndMBB )
     //
     // The prefetch is used only for MVC.  The JLH is used only for CLC.
     MBB = LoopMBB;
     BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisDestReg)
       .addReg(StartDestReg).addMBB(StartMBB)
       .addReg(NextDestReg).addMBB(NextMBB);
     if (!HaveSingleBase)
       BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisSrcReg)
         .addReg(StartSrcReg).addMBB(StartMBB)
         .addReg(NextSrcReg).addMBB(NextMBB);
     BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisCountReg)
       .addReg(StartCountReg).addMBB(StartMBB)
       .addReg(NextCountReg).addMBB(NextMBB);
     if (Opcode == SystemZ::MVC)
       BuildMI(MBB, DL, TII->get(SystemZ::PFD))
         .addImm(SystemZ::PFD_WRITE)
         .addReg(ThisDestReg).addImm(DestDisp - IsMemset + 768).addReg(0);
     insertMemMemOp(MBB, MBB->end(),
                    MachineOperand::CreateReg(ThisDestReg, false), DestDisp,
                    MachineOperand::CreateReg(ThisSrcReg, false), SrcDisp, 256);
     if (EndMBB) {
       BuildMI(MBB, DL, TII->get(SystemZ::BRC))
         .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE)
         .addMBB(EndMBB);
       MBB->addSuccessor(EndMBB);
       MBB->addSuccessor(NextMBB);
     }
 
     // NextMBB:
     //   %NextDestReg = LA 256(%ThisDestReg)
     //   %NextSrcReg = LA 256(%ThisSrcReg)
     //   %NextCountReg = AGHI %ThisCountReg, -1
     //   CGHI %NextCountReg, 0
     //   JLH LoopMBB
     //   # fall through to DoneMBB
     //
     // The AGHI, CGHI and JLH should be converted to BRCTG by later passes.
     MBB = NextMBB;
     BuildMI(MBB, DL, TII->get(SystemZ::LA), NextDestReg)
       .addReg(ThisDestReg).addImm(256).addReg(0);
     if (!HaveSingleBase)
       BuildMI(MBB, DL, TII->get(SystemZ::LA), NextSrcReg)
         .addReg(ThisSrcReg).addImm(256).addReg(0);
     BuildMI(MBB, DL, TII->get(SystemZ::AGHI), NextCountReg)
       .addReg(ThisCountReg).addImm(-1);
     BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
       .addReg(NextCountReg).addImm(0);
     BuildMI(MBB, DL, TII->get(SystemZ::BRC))
       .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE)
       .addMBB(LoopMBB);
     MBB->addSuccessor(LoopMBB);
     MBB->addSuccessor(DoneMBB);
 
     MBB = DoneMBB;
     if (IsRegForm) {
       // DoneMBB:
       // # Make PHIs for RemDestReg/RemSrcReg as the loop may or may not run.
       // # Use EXecute Relative Long for the remainder of the bytes. The target
       //   instruction of the EXRL will have a length field of 1 since 0 is an
       //   illegal value. The number of bytes processed becomes (%LenAdjReg &
       //   0xff) + 1.
       // # Fall through to AllDoneMBB.
       Register RemSrcReg  = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
       Register RemDestReg = HaveSingleBase ? RemSrcReg
         : MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
       BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemDestReg)
         .addReg(StartDestReg).addMBB(StartMBB)
         .addReg(NextDestReg).addMBB(NextMBB);
       if (!HaveSingleBase)
         BuildMI(MBB, DL, TII->get(SystemZ::PHI), RemSrcReg)
           .addReg(StartSrcReg).addMBB(StartMBB)
           .addReg(NextSrcReg).addMBB(NextMBB);
       if (IsMemset)
         insertMemMemOp(MBB, MBB->end(),
                        MachineOperand::CreateReg(RemDestReg, false), DestDisp,
                        MachineOperand::CreateReg(RemSrcReg, false), SrcDisp, 1);
       MachineInstrBuilder EXRL_MIB =
         BuildMI(MBB, DL, TII->get(SystemZ::EXRL_Pseudo))
           .addImm(Opcode)
           .addReg(LenAdjReg)
           .addReg(RemDestReg).addImm(DestDisp)
           .addReg(RemSrcReg).addImm(SrcDisp);
       MBB->addSuccessor(AllDoneMBB);
       MBB = AllDoneMBB;
       if (EndMBB) {
         EXRL_MIB.addReg(SystemZ::CC, RegState::ImplicitDefine);
         MBB->addLiveIn(SystemZ::CC);
       }
     }
   }
 
   // Handle any remaining bytes with straight-line code.
   while (ImmLength > 0) {
     uint64_t ThisLength = std::min(ImmLength, uint64_t(256));
     // The previous iteration might have created out-of-range displacements.
     // Apply them using LA/LAY if so.
     foldDisplIfNeeded(DestBase, DestDisp);
     foldDisplIfNeeded(SrcBase, SrcDisp);
     insertMemMemOp(MBB, MI, DestBase, DestDisp, SrcBase, SrcDisp, ThisLength);
     DestDisp += ThisLength;
     SrcDisp += ThisLength;
     ImmLength -= ThisLength;
     // If there's another CLC to go, branch to the end if a difference
     // was found.
     if (EndMBB && ImmLength > 0) {
       MachineBasicBlock *NextMBB = SystemZ::splitBlockBefore(MI, MBB);
       BuildMI(MBB, DL, TII->get(SystemZ::BRC))
         .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE)
         .addMBB(EndMBB);
       MBB->addSuccessor(EndMBB);
       MBB->addSuccessor(NextMBB);
       MBB = NextMBB;
     }
   }
   if (EndMBB) {
     MBB->addSuccessor(EndMBB);
     MBB = EndMBB;
     MBB->addLiveIn(SystemZ::CC);
   }
 
   MI.eraseFromParent();
   return MBB;
 }
 
 // Decompose string pseudo-instruction MI into a loop that continually performs
 // Opcode until CC != 3.
 MachineBasicBlock *SystemZTargetLowering::emitStringWrapper(
     MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
   MachineFunction &MF = *MBB->getParent();
   const SystemZInstrInfo *TII =
       static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
   MachineRegisterInfo &MRI = MF.getRegInfo();
   DebugLoc DL = MI.getDebugLoc();
 
   uint64_t End1Reg = MI.getOperand(0).getReg();
   uint64_t Start1Reg = MI.getOperand(1).getReg();
   uint64_t Start2Reg = MI.getOperand(2).getReg();
   uint64_t CharReg = MI.getOperand(3).getReg();
 
   const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass;
   uint64_t This1Reg = MRI.createVirtualRegister(RC);
   uint64_t This2Reg = MRI.createVirtualRegister(RC);
   uint64_t End2Reg  = MRI.createVirtualRegister(RC);
 
   MachineBasicBlock *StartMBB = MBB;
   MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MI, MBB);
   MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(StartMBB);
 
   //  StartMBB:
   //   # fall through to LoopMBB
   MBB->addSuccessor(LoopMBB);
 
   //  LoopMBB:
   //   %This1Reg = phi [ %Start1Reg, StartMBB ], [ %End1Reg, LoopMBB ]
   //   %This2Reg = phi [ %Start2Reg, StartMBB ], [ %End2Reg, LoopMBB ]
   //   R0L = %CharReg
   //   %End1Reg, %End2Reg = CLST %This1Reg, %This2Reg -- uses R0L
   //   JO LoopMBB
   //   # fall through to DoneMBB
   //
   // The load of R0L can be hoisted by post-RA LICM.
   MBB = LoopMBB;
 
   BuildMI(MBB, DL, TII->get(SystemZ::PHI), This1Reg)
     .addReg(Start1Reg).addMBB(StartMBB)
     .addReg(End1Reg).addMBB(LoopMBB);
   BuildMI(MBB, DL, TII->get(SystemZ::PHI), This2Reg)
     .addReg(Start2Reg).addMBB(StartMBB)
     .addReg(End2Reg).addMBB(LoopMBB);
   BuildMI(MBB, DL, TII->get(TargetOpcode::COPY), SystemZ::R0L).addReg(CharReg);
   BuildMI(MBB, DL, TII->get(Opcode))
     .addReg(End1Reg, RegState::Define).addReg(End2Reg, RegState::Define)
     .addReg(This1Reg).addReg(This2Reg);
   BuildMI(MBB, DL, TII->get(SystemZ::BRC))
     .addImm(SystemZ::CCMASK_ANY).addImm(SystemZ::CCMASK_3).addMBB(LoopMBB);
   MBB->addSuccessor(LoopMBB);
   MBB->addSuccessor(DoneMBB);
 
   DoneMBB->addLiveIn(SystemZ::CC);
 
   MI.eraseFromParent();
   return DoneMBB;
 }
 
 // Update TBEGIN instruction with final opcode and register clobbers.
 MachineBasicBlock *SystemZTargetLowering::emitTransactionBegin(
     MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode,
     bool NoFloat) const {
   MachineFunction &MF = *MBB->getParent();
   const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
   const SystemZInstrInfo *TII = Subtarget.getInstrInfo();
 
   // Update opcode.
   MI.setDesc(TII->get(Opcode));
 
   // We cannot handle a TBEGIN that clobbers the stack or frame pointer.
   // Make sure to add the corresponding GRSM bits if they are missing.
   uint64_t Control = MI.getOperand(2).getImm();
   static const unsigned GPRControlBit[16] = {
     0x8000, 0x8000, 0x4000, 0x4000, 0x2000, 0x2000, 0x1000, 0x1000,
     0x0800, 0x0800, 0x0400, 0x0400, 0x0200, 0x0200, 0x0100, 0x0100
   };
   Control |= GPRControlBit[15];
   if (TFI->hasFP(MF))
     Control |= GPRControlBit[11];
   MI.getOperand(2).setImm(Control);
 
   // Add GPR clobbers.
   for (int I = 0; I < 16; I++) {
     if ((Control & GPRControlBit[I]) == 0) {
       unsigned Reg = SystemZMC::GR64Regs[I];
       MI.addOperand(MachineOperand::CreateReg(Reg, true, true));
     }
   }
 
   // Add FPR/VR clobbers.
   if (!NoFloat && (Control & 4) != 0) {
     if (Subtarget.hasVector()) {
       for (unsigned Reg : SystemZMC::VR128Regs) {
         MI.addOperand(MachineOperand::CreateReg(Reg, true, true));
       }
     } else {
       for (unsigned Reg : SystemZMC::FP64Regs) {
         MI.addOperand(MachineOperand::CreateReg(Reg, true, true));
       }
     }
   }
 
   return MBB;
 }
 
 MachineBasicBlock *SystemZTargetLowering::emitLoadAndTestCmp0(
     MachineInstr &MI, MachineBasicBlock *MBB, unsigned Opcode) const {
   MachineFunction &MF = *MBB->getParent();
   MachineRegisterInfo *MRI = &MF.getRegInfo();
   const SystemZInstrInfo *TII =
       static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
   DebugLoc DL = MI.getDebugLoc();
 
   Register SrcReg = MI.getOperand(0).getReg();
 
   // Create new virtual register of the same class as source.
   const TargetRegisterClass *RC = MRI->getRegClass(SrcReg);
   Register DstReg = MRI->createVirtualRegister(RC);
 
   // Replace pseudo with a normal load-and-test that models the def as
   // well.
   BuildMI(*MBB, MI, DL, TII->get(Opcode), DstReg)
     .addReg(SrcReg)
     .setMIFlags(MI.getFlags());
   MI.eraseFromParent();
 
   return MBB;
 }
 
 MachineBasicBlock *SystemZTargetLowering::emitProbedAlloca(
     MachineInstr &MI, MachineBasicBlock *MBB) const {
   MachineFunction &MF = *MBB->getParent();
   MachineRegisterInfo *MRI = &MF.getRegInfo();
   const SystemZInstrInfo *TII =
       static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
   DebugLoc DL = MI.getDebugLoc();
   const unsigned ProbeSize = getStackProbeSize(MF);
   Register DstReg = MI.getOperand(0).getReg();
   Register SizeReg = MI.getOperand(2).getReg();
 
   MachineBasicBlock *StartMBB = MBB;
   MachineBasicBlock *DoneMBB  = SystemZ::splitBlockAfter(MI, MBB);
   MachineBasicBlock *LoopTestMBB  = SystemZ::emitBlockAfter(StartMBB);
   MachineBasicBlock *LoopBodyMBB = SystemZ::emitBlockAfter(LoopTestMBB);
   MachineBasicBlock *TailTestMBB = SystemZ::emitBlockAfter(LoopBodyMBB);
   MachineBasicBlock *TailMBB = SystemZ::emitBlockAfter(TailTestMBB);
 
   MachineMemOperand *VolLdMMO = MF.getMachineMemOperand(MachinePointerInfo(),
     MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad, 8, Align(1));
 
   Register PHIReg = MRI->createVirtualRegister(&SystemZ::ADDR64BitRegClass);
   Register IncReg = MRI->createVirtualRegister(&SystemZ::ADDR64BitRegClass);
 
   //  LoopTestMBB
   //  BRC TailTestMBB
   //  # fallthrough to LoopBodyMBB
   StartMBB->addSuccessor(LoopTestMBB);
   MBB = LoopTestMBB;
   BuildMI(MBB, DL, TII->get(SystemZ::PHI), PHIReg)
     .addReg(SizeReg)
     .addMBB(StartMBB)
     .addReg(IncReg)
     .addMBB(LoopBodyMBB);
   BuildMI(MBB, DL, TII->get(SystemZ::CLGFI))
     .addReg(PHIReg)
     .addImm(ProbeSize);
   BuildMI(MBB, DL, TII->get(SystemZ::BRC))
     .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_LT)
     .addMBB(TailTestMBB);
   MBB->addSuccessor(LoopBodyMBB);
   MBB->addSuccessor(TailTestMBB);
 
   //  LoopBodyMBB: Allocate and probe by means of a volatile compare.
   //  J LoopTestMBB
   MBB = LoopBodyMBB;
   BuildMI(MBB, DL, TII->get(SystemZ::SLGFI), IncReg)
     .addReg(PHIReg)
     .addImm(ProbeSize);
   BuildMI(MBB, DL, TII->get(SystemZ::SLGFI), SystemZ::R15D)
     .addReg(SystemZ::R15D)
     .addImm(ProbeSize);
   BuildMI(MBB, DL, TII->get(SystemZ::CG)).addReg(SystemZ::R15D)
     .addReg(SystemZ::R15D).addImm(ProbeSize - 8).addReg(0)
     .setMemRefs(VolLdMMO);
   BuildMI(MBB, DL, TII->get(SystemZ::J)).addMBB(LoopTestMBB);
   MBB->addSuccessor(LoopTestMBB);
 
   //  TailTestMBB
   //  BRC DoneMBB
   //  # fallthrough to TailMBB
   MBB = TailTestMBB;
   BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
     .addReg(PHIReg)
     .addImm(0);
   BuildMI(MBB, DL, TII->get(SystemZ::BRC))
     .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_EQ)
     .addMBB(DoneMBB);
   MBB->addSuccessor(TailMBB);
   MBB->addSuccessor(DoneMBB);
 
   //  TailMBB
   //  # fallthrough to DoneMBB
   MBB = TailMBB;
   BuildMI(MBB, DL, TII->get(SystemZ::SLGR), SystemZ::R15D)
     .addReg(SystemZ::R15D)
     .addReg(PHIReg);
   BuildMI(MBB, DL, TII->get(SystemZ::CG)).addReg(SystemZ::R15D)
     .addReg(SystemZ::R15D).addImm(-8).addReg(PHIReg)
     .setMemRefs(VolLdMMO);
   MBB->addSuccessor(DoneMBB);
 
   //  DoneMBB
   MBB = DoneMBB;
   BuildMI(*MBB, MBB->begin(), DL, TII->get(TargetOpcode::COPY), DstReg)
     .addReg(SystemZ::R15D);
 
   MI.eraseFromParent();
   return DoneMBB;
 }
 
 SDValue SystemZTargetLowering::
 getBackchainAddress(SDValue SP, SelectionDAG &DAG) const {
   MachineFunction &MF = DAG.getMachineFunction();
   auto *TFL = Subtarget.getFrameLowering<SystemZELFFrameLowering>();
   SDLoc DL(SP);
   return DAG.getNode(ISD::ADD, DL, MVT::i64, SP,
                      DAG.getIntPtrConstant(TFL->getBackchainOffset(MF), DL));
 }
 
 MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter(
     MachineInstr &MI, MachineBasicBlock *MBB) const {
   switch (MI.getOpcode()) {
   case SystemZ::Select32:
   case SystemZ::Select64:
   case SystemZ::SelectF32:
   case SystemZ::SelectF64:
   case SystemZ::SelectF128:
   case SystemZ::SelectVR32:
   case SystemZ::SelectVR64:
   case SystemZ::SelectVR128:
     return emitSelect(MI, MBB);
 
   case SystemZ::CondStore8Mux:
     return emitCondStore(MI, MBB, SystemZ::STCMux, 0, false);
   case SystemZ::CondStore8MuxInv:
     return emitCondStore(MI, MBB, SystemZ::STCMux, 0, true);
   case SystemZ::CondStore16Mux:
     return emitCondStore(MI, MBB, SystemZ::STHMux, 0, false);
   case SystemZ::CondStore16MuxInv:
     return emitCondStore(MI, MBB, SystemZ::STHMux, 0, true);
   case SystemZ::CondStore32Mux:
     return emitCondStore(MI, MBB, SystemZ::STMux, SystemZ::STOCMux, false);
   case SystemZ::CondStore32MuxInv:
     return emitCondStore(MI, MBB, SystemZ::STMux, SystemZ::STOCMux, true);
   case SystemZ::CondStore8:
     return emitCondStore(MI, MBB, SystemZ::STC, 0, false);
   case SystemZ::CondStore8Inv:
     return emitCondStore(MI, MBB, SystemZ::STC, 0, true);
   case SystemZ::CondStore16:
     return emitCondStore(MI, MBB, SystemZ::STH, 0, false);
   case SystemZ::CondStore16Inv:
     return emitCondStore(MI, MBB, SystemZ::STH, 0, true);
   case SystemZ::CondStore32:
     return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, false);
   case SystemZ::CondStore32Inv:
     return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, true);
   case SystemZ::CondStore64:
     return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, false);
   case SystemZ::CondStore64Inv:
     return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, true);
   case SystemZ::CondStoreF32:
     return emitCondStore(MI, MBB, SystemZ::STE, 0, false);
   case SystemZ::CondStoreF32Inv:
     return emitCondStore(MI, MBB, SystemZ::STE, 0, true);
   case SystemZ::CondStoreF64:
     return emitCondStore(MI, MBB, SystemZ::STD, 0, false);
   case SystemZ::CondStoreF64Inv:
     return emitCondStore(MI, MBB, SystemZ::STD, 0, true);
 
   case SystemZ::PAIR128:
     return emitPair128(MI, MBB);
   case SystemZ::AEXT128:
     return emitExt128(MI, MBB, false);
   case SystemZ::ZEXT128:
     return emitExt128(MI, MBB, true);
 
   case SystemZ::ATOMIC_SWAPW:
     return emitAtomicLoadBinary(MI, MBB, 0, 0);
   case SystemZ::ATOMIC_SWAP_32:
     return emitAtomicLoadBinary(MI, MBB, 0, 32);
   case SystemZ::ATOMIC_SWAP_64:
     return emitAtomicLoadBinary(MI, MBB, 0, 64);
 
   case SystemZ::ATOMIC_LOADW_AR:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 0);
   case SystemZ::ATOMIC_LOADW_AFI:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 0);
   case SystemZ::ATOMIC_LOAD_AR:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 32);
   case SystemZ::ATOMIC_LOAD_AHI:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::AHI, 32);
   case SystemZ::ATOMIC_LOAD_AFI:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 32);
   case SystemZ::ATOMIC_LOAD_AGR:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::AGR, 64);
   case SystemZ::ATOMIC_LOAD_AGHI:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::AGHI, 64);
   case SystemZ::ATOMIC_LOAD_AGFI:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::AGFI, 64);
 
   case SystemZ::ATOMIC_LOADW_SR:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 0);
   case SystemZ::ATOMIC_LOAD_SR:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 32);
   case SystemZ::ATOMIC_LOAD_SGR:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::SGR, 64);
 
   case SystemZ::ATOMIC_LOADW_NR:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0);
   case SystemZ::ATOMIC_LOADW_NILH:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0);
   case SystemZ::ATOMIC_LOAD_NR:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32);
   case SystemZ::ATOMIC_LOAD_NILL:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32);
   case SystemZ::ATOMIC_LOAD_NILH:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32);
   case SystemZ::ATOMIC_LOAD_NILF:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32);
   case SystemZ::ATOMIC_LOAD_NGR:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64);
   case SystemZ::ATOMIC_LOAD_NILL64:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64);
   case SystemZ::ATOMIC_LOAD_NILH64:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64);
   case SystemZ::ATOMIC_LOAD_NIHL64:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64);
   case SystemZ::ATOMIC_LOAD_NIHH64:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64);
   case SystemZ::ATOMIC_LOAD_NILF64:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64);
   case SystemZ::ATOMIC_LOAD_NIHF64:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64);
 
   case SystemZ::ATOMIC_LOADW_OR:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 0);
   case SystemZ::ATOMIC_LOADW_OILH:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 0);
   case SystemZ::ATOMIC_LOAD_OR:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 32);
   case SystemZ::ATOMIC_LOAD_OILL:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL, 32);
   case SystemZ::ATOMIC_LOAD_OILH:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 32);
   case SystemZ::ATOMIC_LOAD_OILF:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF, 32);
   case SystemZ::ATOMIC_LOAD_OGR:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::OGR, 64);
   case SystemZ::ATOMIC_LOAD_OILL64:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL64, 64);
   case SystemZ::ATOMIC_LOAD_OILH64:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH64, 64);
   case SystemZ::ATOMIC_LOAD_OIHL64:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHL64, 64);
   case SystemZ::ATOMIC_LOAD_OIHH64:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHH64, 64);
   case SystemZ::ATOMIC_LOAD_OILF64:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF64, 64);
   case SystemZ::ATOMIC_LOAD_OIHF64:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHF64, 64);
 
   case SystemZ::ATOMIC_LOADW_XR:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 0);
   case SystemZ::ATOMIC_LOADW_XILF:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 0);
   case SystemZ::ATOMIC_LOAD_XR:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 32);
   case SystemZ::ATOMIC_LOAD_XILF:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 32);
   case SystemZ::ATOMIC_LOAD_XGR:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::XGR, 64);
   case SystemZ::ATOMIC_LOAD_XILF64:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF64, 64);
   case SystemZ::ATOMIC_LOAD_XIHF64:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::XIHF64, 64);
 
   case SystemZ::ATOMIC_LOADW_NRi:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0, true);
   case SystemZ::ATOMIC_LOADW_NILHi:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0, true);
   case SystemZ::ATOMIC_LOAD_NRi:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32, true);
   case SystemZ::ATOMIC_LOAD_NILLi:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32, true);
   case SystemZ::ATOMIC_LOAD_NILHi:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32, true);
   case SystemZ::ATOMIC_LOAD_NILFi:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32, true);
   case SystemZ::ATOMIC_LOAD_NGRi:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64, true);
   case SystemZ::ATOMIC_LOAD_NILL64i:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64, true);
   case SystemZ::ATOMIC_LOAD_NILH64i:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64, true);
   case SystemZ::ATOMIC_LOAD_NIHL64i:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64, true);
   case SystemZ::ATOMIC_LOAD_NIHH64i:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64, true);
   case SystemZ::ATOMIC_LOAD_NILF64i:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64, true);
   case SystemZ::ATOMIC_LOAD_NIHF64i:
     return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64, true);
 
   case SystemZ::ATOMIC_LOADW_MIN:
     return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR,
                                 SystemZ::CCMASK_CMP_LE, 0);
   case SystemZ::ATOMIC_LOAD_MIN_32:
     return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR,
                                 SystemZ::CCMASK_CMP_LE, 32);
   case SystemZ::ATOMIC_LOAD_MIN_64:
     return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR,
                                 SystemZ::CCMASK_CMP_LE, 64);
 
   case SystemZ::ATOMIC_LOADW_MAX:
     return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR,
                                 SystemZ::CCMASK_CMP_GE, 0);
   case SystemZ::ATOMIC_LOAD_MAX_32:
     return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR,
                                 SystemZ::CCMASK_CMP_GE, 32);
   case SystemZ::ATOMIC_LOAD_MAX_64:
     return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR,
                                 SystemZ::CCMASK_CMP_GE, 64);
 
   case SystemZ::ATOMIC_LOADW_UMIN:
     return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR,
                                 SystemZ::CCMASK_CMP_LE, 0);
   case SystemZ::ATOMIC_LOAD_UMIN_32:
     return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR,
                                 SystemZ::CCMASK_CMP_LE, 32);
   case SystemZ::ATOMIC_LOAD_UMIN_64:
     return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR,
                                 SystemZ::CCMASK_CMP_LE, 64);
 
   case SystemZ::ATOMIC_LOADW_UMAX:
     return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR,
                                 SystemZ::CCMASK_CMP_GE, 0);
   case SystemZ::ATOMIC_LOAD_UMAX_32:
     return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR,
                                 SystemZ::CCMASK_CMP_GE, 32);
   case SystemZ::ATOMIC_LOAD_UMAX_64:
     return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR,
                                 SystemZ::CCMASK_CMP_GE, 64);
 
   case SystemZ::ATOMIC_CMP_SWAPW:
     return emitAtomicCmpSwapW(MI, MBB);
   case SystemZ::MVCImm:
   case SystemZ::MVCReg:
     return emitMemMemWrapper(MI, MBB, SystemZ::MVC);
   case SystemZ::NCImm:
     return emitMemMemWrapper(MI, MBB, SystemZ::NC);
   case SystemZ::OCImm:
     return emitMemMemWrapper(MI, MBB, SystemZ::OC);
   case SystemZ::XCImm:
   case SystemZ::XCReg:
     return emitMemMemWrapper(MI, MBB, SystemZ::XC);
   case SystemZ::CLCImm:
   case SystemZ::CLCReg:
     return emitMemMemWrapper(MI, MBB, SystemZ::CLC);
   case SystemZ::MemsetImmImm:
   case SystemZ::MemsetImmReg:
   case SystemZ::MemsetRegImm:
   case SystemZ::MemsetRegReg:
     return emitMemMemWrapper(MI, MBB, SystemZ::MVC, true/*IsMemset*/);
   case SystemZ::CLSTLoop:
     return emitStringWrapper(MI, MBB, SystemZ::CLST);
   case SystemZ::MVSTLoop:
     return emitStringWrapper(MI, MBB, SystemZ::MVST);
   case SystemZ::SRSTLoop:
     return emitStringWrapper(MI, MBB, SystemZ::SRST);
   case SystemZ::TBEGIN:
     return emitTransactionBegin(MI, MBB, SystemZ::TBEGIN, false);
   case SystemZ::TBEGIN_nofloat:
     return emitTransactionBegin(MI, MBB, SystemZ::TBEGIN, true);
   case SystemZ::TBEGINC:
     return emitTransactionBegin(MI, MBB, SystemZ::TBEGINC, true);
   case SystemZ::LTEBRCompare_VecPseudo:
     return emitLoadAndTestCmp0(MI, MBB, SystemZ::LTEBR);
   case SystemZ::LTDBRCompare_VecPseudo:
     return emitLoadAndTestCmp0(MI, MBB, SystemZ::LTDBR);
   case SystemZ::LTXBRCompare_VecPseudo:
     return emitLoadAndTestCmp0(MI, MBB, SystemZ::LTXBR);
 
   case SystemZ::PROBED_ALLOCA:
     return emitProbedAlloca(MI, MBB);
 
   case TargetOpcode::STACKMAP:
   case TargetOpcode::PATCHPOINT:
     return emitPatchPoint(MI, MBB);
 
   default:
     llvm_unreachable("Unexpected instr type to insert");
   }
 }
 
 // This is only used by the isel schedulers, and is needed only to prevent
 // compiler from crashing when list-ilp is used.
 const TargetRegisterClass *
 SystemZTargetLowering::getRepRegClassFor(MVT VT) const {
   if (VT == MVT::Untyped)
     return &SystemZ::ADDR128BitRegClass;
   return TargetLowering::getRepRegClassFor(VT);
 }
diff --git a/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 2aab79e89078..7eaa28bd1320 100644
--- a/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/contrib/llvm-project/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -1,3795 +1,3799 @@
 //===- InstCombineAndOrXor.cpp --------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the visitAnd, visitOr, and visitXor functions.
 //
 //===----------------------------------------------------------------------===//
 
 #include "InstCombineInternal.h"
 #include "llvm/Analysis/CmpInstAnalysis.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include "llvm/Transforms/Utils/Local.h"
 
 using namespace llvm;
 using namespace PatternMatch;
 
 #define DEBUG_TYPE "instcombine"
 
 /// Similar to getICmpCode but for FCmpInst. This encodes a fcmp predicate into
 /// a four bit mask.
 static unsigned getFCmpCode(FCmpInst::Predicate CC) {
   assert(FCmpInst::FCMP_FALSE <= CC && CC <= FCmpInst::FCMP_TRUE &&
          "Unexpected FCmp predicate!");
   // Take advantage of the bit pattern of FCmpInst::Predicate here.
   //                                                 U L G E
   static_assert(FCmpInst::FCMP_FALSE ==  0, "");  // 0 0 0 0
   static_assert(FCmpInst::FCMP_OEQ   ==  1, "");  // 0 0 0 1
   static_assert(FCmpInst::FCMP_OGT   ==  2, "");  // 0 0 1 0
   static_assert(FCmpInst::FCMP_OGE   ==  3, "");  // 0 0 1 1
   static_assert(FCmpInst::FCMP_OLT   ==  4, "");  // 0 1 0 0
   static_assert(FCmpInst::FCMP_OLE   ==  5, "");  // 0 1 0 1
   static_assert(FCmpInst::FCMP_ONE   ==  6, "");  // 0 1 1 0
   static_assert(FCmpInst::FCMP_ORD   ==  7, "");  // 0 1 1 1
   static_assert(FCmpInst::FCMP_UNO   ==  8, "");  // 1 0 0 0
   static_assert(FCmpInst::FCMP_UEQ   ==  9, "");  // 1 0 0 1
   static_assert(FCmpInst::FCMP_UGT   == 10, "");  // 1 0 1 0
   static_assert(FCmpInst::FCMP_UGE   == 11, "");  // 1 0 1 1
   static_assert(FCmpInst::FCMP_ULT   == 12, "");  // 1 1 0 0
   static_assert(FCmpInst::FCMP_ULE   == 13, "");  // 1 1 0 1
   static_assert(FCmpInst::FCMP_UNE   == 14, "");  // 1 1 1 0
   static_assert(FCmpInst::FCMP_TRUE  == 15, "");  // 1 1 1 1
   return CC;
 }
 
 /// This is the complement of getICmpCode, which turns an opcode and two
 /// operands into either a constant true or false, or a brand new ICmp
 /// instruction. The sign is passed in to determine which kind of predicate to
 /// use in the new icmp instruction.
 static Value *getNewICmpValue(unsigned Code, bool Sign, Value *LHS, Value *RHS,
                               InstCombiner::BuilderTy &Builder) {
   ICmpInst::Predicate NewPred;
   if (Constant *TorF = getPredForICmpCode(Code, Sign, LHS->getType(), NewPred))
     return TorF;
   return Builder.CreateICmp(NewPred, LHS, RHS);
 }
 
 /// This is the complement of getFCmpCode, which turns an opcode and two
 /// operands into either a FCmp instruction, or a true/false constant.
 static Value *getFCmpValue(unsigned Code, Value *LHS, Value *RHS,
                            InstCombiner::BuilderTy &Builder) {
   const auto Pred = static_cast<FCmpInst::Predicate>(Code);
   assert(FCmpInst::FCMP_FALSE <= Pred && Pred <= FCmpInst::FCMP_TRUE &&
          "Unexpected FCmp predicate!");
   if (Pred == FCmpInst::FCMP_FALSE)
     return ConstantInt::get(CmpInst::makeCmpResultType(LHS->getType()), 0);
   if (Pred == FCmpInst::FCMP_TRUE)
     return ConstantInt::get(CmpInst::makeCmpResultType(LHS->getType()), 1);
   return Builder.CreateFCmp(Pred, LHS, RHS);
 }
 
 /// Transform BITWISE_OP(BSWAP(A),BSWAP(B)) or
 /// BITWISE_OP(BSWAP(A), Constant) to BSWAP(BITWISE_OP(A, B))
 /// \param I Binary operator to transform.
 /// \return Pointer to node that must replace the original binary operator, or
 ///         null pointer if no transformation was made.
 static Value *SimplifyBSwap(BinaryOperator &I,
                             InstCombiner::BuilderTy &Builder) {
   assert(I.isBitwiseLogicOp() && "Unexpected opcode for bswap simplifying");
 
   Value *OldLHS = I.getOperand(0);
   Value *OldRHS = I.getOperand(1);
 
   Value *NewLHS;
   if (!match(OldLHS, m_BSwap(m_Value(NewLHS))))
     return nullptr;
 
   Value *NewRHS;
   const APInt *C;
 
   if (match(OldRHS, m_BSwap(m_Value(NewRHS)))) {
     // OP( BSWAP(x), BSWAP(y) ) -> BSWAP( OP(x, y) )
     if (!OldLHS->hasOneUse() && !OldRHS->hasOneUse())
       return nullptr;
     // NewRHS initialized by the matcher.
   } else if (match(OldRHS, m_APInt(C))) {
     // OP( BSWAP(x), CONSTANT ) -> BSWAP( OP(x, BSWAP(CONSTANT) ) )
     if (!OldLHS->hasOneUse())
       return nullptr;
     NewRHS = ConstantInt::get(I.getType(), C->byteSwap());
   } else
     return nullptr;
 
   Value *BinOp = Builder.CreateBinOp(I.getOpcode(), NewLHS, NewRHS);
   Function *F = Intrinsic::getDeclaration(I.getModule(), Intrinsic::bswap,
                                           I.getType());
   return Builder.CreateCall(F, BinOp);
 }
 
 /// Emit a computation of: (V >= Lo && V < Hi) if Inside is true, otherwise
 /// (V < Lo || V >= Hi). This method expects that Lo < Hi. IsSigned indicates
 /// whether to treat V, Lo, and Hi as signed or not.
 Value *InstCombinerImpl::insertRangeTest(Value *V, const APInt &Lo,
                                          const APInt &Hi, bool isSigned,
                                          bool Inside) {
   assert((isSigned ? Lo.slt(Hi) : Lo.ult(Hi)) &&
          "Lo is not < Hi in range emission code!");
 
   Type *Ty = V->getType();
 
   // V >= Min && V <  Hi --> V <  Hi
   // V <  Min || V >= Hi --> V >= Hi
   ICmpInst::Predicate Pred = Inside ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_UGE;
   if (isSigned ? Lo.isMinSignedValue() : Lo.isMinValue()) {
     Pred = isSigned ? ICmpInst::getSignedPredicate(Pred) : Pred;
     return Builder.CreateICmp(Pred, V, ConstantInt::get(Ty, Hi));
   }
 
   // V >= Lo && V <  Hi --> V - Lo u<  Hi - Lo
   // V <  Lo || V >= Hi --> V - Lo u>= Hi - Lo
   Value *VMinusLo =
       Builder.CreateSub(V, ConstantInt::get(Ty, Lo), V->getName() + ".off");
   Constant *HiMinusLo = ConstantInt::get(Ty, Hi - Lo);
   return Builder.CreateICmp(Pred, VMinusLo, HiMinusLo);
 }
 
 /// Classify (icmp eq (A & B), C) and (icmp ne (A & B), C) as matching patterns
 /// that can be simplified.
 /// One of A and B is considered the mask. The other is the value. This is
 /// described as the "AMask" or "BMask" part of the enum. If the enum contains
 /// only "Mask", then both A and B can be considered masks. If A is the mask,
 /// then it was proven that (A & C) == C. This is trivial if C == A or C == 0.
 /// If both A and C are constants, this proof is also easy.
 /// For the following explanations, we assume that A is the mask.
 ///
 /// "AllOnes" declares that the comparison is true only if (A & B) == A or all
 /// bits of A are set in B.
 ///   Example: (icmp eq (A & 3), 3) -> AMask_AllOnes
 ///
 /// "AllZeros" declares that the comparison is true only if (A & B) == 0 or all
 /// bits of A are cleared in B.
 ///   Example: (icmp eq (A & 3), 0) -> Mask_AllZeroes
 ///
 /// "Mixed" declares that (A & B) == C and C might or might not contain any
 /// number of one bits and zero bits.
 ///   Example: (icmp eq (A & 3), 1) -> AMask_Mixed
 ///
 /// "Not" means that in above descriptions "==" should be replaced by "!=".
 ///   Example: (icmp ne (A & 3), 3) -> AMask_NotAllOnes
 ///
 /// If the mask A contains a single bit, then the following is equivalent:
 ///    (icmp eq (A & B), A) equals (icmp ne (A & B), 0)
 ///    (icmp ne (A & B), A) equals (icmp eq (A & B), 0)
 enum MaskedICmpType {
   AMask_AllOnes           =     1,
   AMask_NotAllOnes        =     2,
   BMask_AllOnes           =     4,
   BMask_NotAllOnes        =     8,
   Mask_AllZeros           =    16,
   Mask_NotAllZeros        =    32,
   AMask_Mixed             =    64,
   AMask_NotMixed          =   128,
   BMask_Mixed             =   256,
   BMask_NotMixed          =   512
 };
 
 /// Return the set of patterns (from MaskedICmpType) that (icmp SCC (A & B), C)
 /// satisfies.
 static unsigned getMaskedICmpType(Value *A, Value *B, Value *C,
                                   ICmpInst::Predicate Pred) {
   const APInt *ConstA = nullptr, *ConstB = nullptr, *ConstC = nullptr;
   match(A, m_APInt(ConstA));
   match(B, m_APInt(ConstB));
   match(C, m_APInt(ConstC));
   bool IsEq = (Pred == ICmpInst::ICMP_EQ);
   bool IsAPow2 = ConstA && ConstA->isPowerOf2();
   bool IsBPow2 = ConstB && ConstB->isPowerOf2();
   unsigned MaskVal = 0;
   if (ConstC && ConstC->isZero()) {
     // if C is zero, then both A and B qualify as mask
     MaskVal |= (IsEq ? (Mask_AllZeros | AMask_Mixed | BMask_Mixed)
                      : (Mask_NotAllZeros | AMask_NotMixed | BMask_NotMixed));
     if (IsAPow2)
       MaskVal |= (IsEq ? (AMask_NotAllOnes | AMask_NotMixed)
                        : (AMask_AllOnes | AMask_Mixed));
     if (IsBPow2)
       MaskVal |= (IsEq ? (BMask_NotAllOnes | BMask_NotMixed)
                        : (BMask_AllOnes | BMask_Mixed));
     return MaskVal;
   }
 
   if (A == C) {
     MaskVal |= (IsEq ? (AMask_AllOnes | AMask_Mixed)
                      : (AMask_NotAllOnes | AMask_NotMixed));
     if (IsAPow2)
       MaskVal |= (IsEq ? (Mask_NotAllZeros | AMask_NotMixed)
                        : (Mask_AllZeros | AMask_Mixed));
   } else if (ConstA && ConstC && ConstC->isSubsetOf(*ConstA)) {
     MaskVal |= (IsEq ? AMask_Mixed : AMask_NotMixed);
   }
 
   if (B == C) {
     MaskVal |= (IsEq ? (BMask_AllOnes | BMask_Mixed)
                      : (BMask_NotAllOnes | BMask_NotMixed));
     if (IsBPow2)
       MaskVal |= (IsEq ? (Mask_NotAllZeros | BMask_NotMixed)
                        : (Mask_AllZeros | BMask_Mixed));
   } else if (ConstB && ConstC && ConstC->isSubsetOf(*ConstB)) {
     MaskVal |= (IsEq ? BMask_Mixed : BMask_NotMixed);
   }
 
   return MaskVal;
 }
 
 /// Convert an analysis of a masked ICmp into its equivalent if all boolean
 /// operations had the opposite sense. Since each "NotXXX" flag (recording !=)
 /// is adjacent to the corresponding normal flag (recording ==), this just
 /// involves swapping those bits over.
 static unsigned conjugateICmpMask(unsigned Mask) {
   unsigned NewMask;
   NewMask = (Mask & (AMask_AllOnes | BMask_AllOnes | Mask_AllZeros |
                      AMask_Mixed | BMask_Mixed))
             << 1;
 
   NewMask |= (Mask & (AMask_NotAllOnes | BMask_NotAllOnes | Mask_NotAllZeros |
                       AMask_NotMixed | BMask_NotMixed))
              >> 1;
 
   return NewMask;
 }
 
 // Adapts the external decomposeBitTestICmp for local use.
 static bool decomposeBitTestICmp(Value *LHS, Value *RHS, CmpInst::Predicate &Pred,
                                  Value *&X, Value *&Y, Value *&Z) {
   APInt Mask;
   if (!llvm::decomposeBitTestICmp(LHS, RHS, Pred, X, Mask))
     return false;
 
   Y = ConstantInt::get(X->getType(), Mask);
   Z = ConstantInt::get(X->getType(), 0);
   return true;
 }
 
 /// Handle (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E).
 /// Return the pattern classes (from MaskedICmpType) for the left hand side and
 /// the right hand side as a pair.
 /// LHS and RHS are the left hand side and the right hand side ICmps and PredL
 /// and PredR are their predicates, respectively.
 static
 Optional<std::pair<unsigned, unsigned>>
 getMaskedTypeForICmpPair(Value *&A, Value *&B, Value *&C,
                          Value *&D, Value *&E, ICmpInst *LHS,
                          ICmpInst *RHS,
                          ICmpInst::Predicate &PredL,
                          ICmpInst::Predicate &PredR) {
   // Don't allow pointers. Splat vectors are fine.
   if (!LHS->getOperand(0)->getType()->isIntOrIntVectorTy() ||
       !RHS->getOperand(0)->getType()->isIntOrIntVectorTy())
     return None;
 
   // Here comes the tricky part:
   // LHS might be of the form L11 & L12 == X, X == L21 & L22,
   // and L11 & L12 == L21 & L22. The same goes for RHS.
   // Now we must find those components L** and R**, that are equal, so
   // that we can extract the parameters A, B, C, D, and E for the canonical
   // above.
   Value *L1 = LHS->getOperand(0);
   Value *L2 = LHS->getOperand(1);
   Value *L11, *L12, *L21, *L22;
   // Check whether the icmp can be decomposed into a bit test.
   if (decomposeBitTestICmp(L1, L2, PredL, L11, L12, L2)) {
     L21 = L22 = L1 = nullptr;
   } else {
     // Look for ANDs in the LHS icmp.
     if (!match(L1, m_And(m_Value(L11), m_Value(L12)))) {
       // Any icmp can be viewed as being trivially masked; if it allows us to
       // remove one, it's worth it.
       L11 = L1;
       L12 = Constant::getAllOnesValue(L1->getType());
     }
 
     if (!match(L2, m_And(m_Value(L21), m_Value(L22)))) {
       L21 = L2;
       L22 = Constant::getAllOnesValue(L2->getType());
     }
   }
 
   // Bail if LHS was a icmp that can't be decomposed into an equality.
   if (!ICmpInst::isEquality(PredL))
     return None;
 
   Value *R1 = RHS->getOperand(0);
   Value *R2 = RHS->getOperand(1);
   Value *R11, *R12;
   bool Ok = false;
   if (decomposeBitTestICmp(R1, R2, PredR, R11, R12, R2)) {
     if (R11 == L11 || R11 == L12 || R11 == L21 || R11 == L22) {
       A = R11;
       D = R12;
     } else if (R12 == L11 || R12 == L12 || R12 == L21 || R12 == L22) {
       A = R12;
       D = R11;
     } else {
       return None;
     }
     E = R2;
     R1 = nullptr;
     Ok = true;
   } else {
     if (!match(R1, m_And(m_Value(R11), m_Value(R12)))) {
       // As before, model no mask as a trivial mask if it'll let us do an
       // optimization.
       R11 = R1;
       R12 = Constant::getAllOnesValue(R1->getType());
     }
 
     if (R11 == L11 || R11 == L12 || R11 == L21 || R11 == L22) {
       A = R11;
       D = R12;
       E = R2;
       Ok = true;
     } else if (R12 == L11 || R12 == L12 || R12 == L21 || R12 == L22) {
       A = R12;
       D = R11;
       E = R2;
       Ok = true;
     }
   }
 
   // Bail if RHS was a icmp that can't be decomposed into an equality.
   if (!ICmpInst::isEquality(PredR))
     return None;
 
   // Look for ANDs on the right side of the RHS icmp.
   if (!Ok) {
     if (!match(R2, m_And(m_Value(R11), m_Value(R12)))) {
       R11 = R2;
       R12 = Constant::getAllOnesValue(R2->getType());
     }
 
     if (R11 == L11 || R11 == L12 || R11 == L21 || R11 == L22) {
       A = R11;
       D = R12;
       E = R1;
       Ok = true;
     } else if (R12 == L11 || R12 == L12 || R12 == L21 || R12 == L22) {
       A = R12;
       D = R11;
       E = R1;
       Ok = true;
     } else {
       return None;
     }
 
     assert(Ok && "Failed to find AND on the right side of the RHS icmp.");
   }
 
   if (L11 == A) {
     B = L12;
     C = L2;
   } else if (L12 == A) {
     B = L11;
     C = L2;
   } else if (L21 == A) {
     B = L22;
     C = L1;
   } else if (L22 == A) {
     B = L21;
     C = L1;
   }
 
   unsigned LeftType = getMaskedICmpType(A, B, C, PredL);
   unsigned RightType = getMaskedICmpType(A, D, E, PredR);
   return Optional<std::pair<unsigned, unsigned>>(std::make_pair(LeftType, RightType));
 }
 
 /// Try to fold (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E) into a single
 /// (icmp(A & X) ==/!= Y), where the left-hand side is of type Mask_NotAllZeros
 /// and the right hand side is of type BMask_Mixed. For example,
 /// (icmp (A & 12) != 0) & (icmp (A & 15) == 8) -> (icmp (A & 15) == 8).
 static Value *foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed(
     ICmpInst *LHS, ICmpInst *RHS, bool IsAnd, Value *A, Value *B, Value *C,
     Value *D, Value *E, ICmpInst::Predicate PredL, ICmpInst::Predicate PredR,
     InstCombiner::BuilderTy &Builder) {
   // We are given the canonical form:
   //   (icmp ne (A & B), 0) & (icmp eq (A & D), E).
   // where D & E == E.
   //
   // If IsAnd is false, we get it in negated form:
   //   (icmp eq (A & B), 0) | (icmp ne (A & D), E) ->
   //      !((icmp ne (A & B), 0) & (icmp eq (A & D), E)).
   //
   // We currently handle the case of B, C, D, E are constant.
   //
   ConstantInt *BCst, *CCst, *DCst, *ECst;
   if (!match(B, m_ConstantInt(BCst)) || !match(C, m_ConstantInt(CCst)) ||
       !match(D, m_ConstantInt(DCst)) || !match(E, m_ConstantInt(ECst)))
     return nullptr;
 
   ICmpInst::Predicate NewCC = IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
 
   // Update E to the canonical form when D is a power of two and RHS is
   // canonicalized as,
   // (icmp ne (A & D), 0) -> (icmp eq (A & D), D) or
   // (icmp ne (A & D), D) -> (icmp eq (A & D), 0).
   if (PredR != NewCC)
     ECst = cast<ConstantInt>(ConstantExpr::getXor(DCst, ECst));
 
   // If B or D is zero, skip because if LHS or RHS can be trivially folded by
   // other folding rules and this pattern won't apply any more.
   if (BCst->getValue() == 0 || DCst->getValue() == 0)
     return nullptr;
 
   // If B and D don't intersect, ie. (B & D) == 0, no folding because we can't
   // deduce anything from it.
   // For example,
   // (icmp ne (A & 12), 0) & (icmp eq (A & 3), 1) -> no folding.
   if ((BCst->getValue() & DCst->getValue()) == 0)
     return nullptr;
 
   // If the following two conditions are met:
   //
   // 1. mask B covers only a single bit that's not covered by mask D, that is,
   // (B & (B ^ D)) is a power of 2 (in other words, B minus the intersection of
   // B and D has only one bit set) and,
   //
   // 2. RHS (and E) indicates that the rest of B's bits are zero (in other
   // words, the intersection of B and D is zero), that is, ((B & D) & E) == 0
   //
   // then that single bit in B must be one and thus the whole expression can be
   // folded to
   //   (A & (B | D)) == (B & (B ^ D)) | E.
   //
   // For example,
   // (icmp ne (A & 12), 0) & (icmp eq (A & 7), 1) -> (icmp eq (A & 15), 9)
   // (icmp ne (A & 15), 0) & (icmp eq (A & 7), 0) -> (icmp eq (A & 15), 8)
   if ((((BCst->getValue() & DCst->getValue()) & ECst->getValue()) == 0) &&
       (BCst->getValue() & (BCst->getValue() ^ DCst->getValue())).isPowerOf2()) {
     APInt BorD = BCst->getValue() | DCst->getValue();
     APInt BandBxorDorE = (BCst->getValue() & (BCst->getValue() ^ DCst->getValue())) |
         ECst->getValue();
     Value *NewMask = ConstantInt::get(BCst->getType(), BorD);
     Value *NewMaskedValue = ConstantInt::get(BCst->getType(), BandBxorDorE);
     Value *NewAnd = Builder.CreateAnd(A, NewMask);
     return Builder.CreateICmp(NewCC, NewAnd, NewMaskedValue);
   }
 
   auto IsSubSetOrEqual = [](ConstantInt *C1, ConstantInt *C2) {
     return (C1->getValue() & C2->getValue()) == C1->getValue();
   };
   auto IsSuperSetOrEqual = [](ConstantInt *C1, ConstantInt *C2) {
     return (C1->getValue() & C2->getValue()) == C2->getValue();
   };
 
   // In the following, we consider only the cases where B is a superset of D, B
   // is a subset of D, or B == D because otherwise there's at least one bit
   // covered by B but not D, in which case we can't deduce much from it, so
   // no folding (aside from the single must-be-one bit case right above.)
   // For example,
   // (icmp ne (A & 14), 0) & (icmp eq (A & 3), 1) -> no folding.
   if (!IsSubSetOrEqual(BCst, DCst) && !IsSuperSetOrEqual(BCst, DCst))
     return nullptr;
 
   // At this point, either B is a superset of D, B is a subset of D or B == D.
 
   // If E is zero, if B is a subset of (or equal to) D, LHS and RHS contradict
   // and the whole expression becomes false (or true if negated), otherwise, no
   // folding.
   // For example,
   // (icmp ne (A & 3), 0) & (icmp eq (A & 7), 0) -> false.
   // (icmp ne (A & 15), 0) & (icmp eq (A & 3), 0) -> no folding.
   if (ECst->isZero()) {
     if (IsSubSetOrEqual(BCst, DCst))
       return ConstantInt::get(LHS->getType(), !IsAnd);
     return nullptr;
   }
 
   // At this point, B, D, E aren't zero and (B & D) == B, (B & D) == D or B ==
   // D. If B is a superset of (or equal to) D, since E is not zero, LHS is
   // subsumed by RHS (RHS implies LHS.) So the whole expression becomes
   // RHS. For example,
   // (icmp ne (A & 255), 0) & (icmp eq (A & 15), 8) -> (icmp eq (A & 15), 8).
   // (icmp ne (A & 15), 0) & (icmp eq (A & 15), 8) -> (icmp eq (A & 15), 8).
   if (IsSuperSetOrEqual(BCst, DCst))
     return RHS;
   // Otherwise, B is a subset of D. If B and E have a common bit set,
   // ie. (B & E) != 0, then LHS is subsumed by RHS. For example.
   // (icmp ne (A & 12), 0) & (icmp eq (A & 15), 8) -> (icmp eq (A & 15), 8).
   assert(IsSubSetOrEqual(BCst, DCst) && "Precondition due to above code");
   if ((BCst->getValue() & ECst->getValue()) != 0)
     return RHS;
   // Otherwise, LHS and RHS contradict and the whole expression becomes false
   // (or true if negated.) For example,
   // (icmp ne (A & 7), 0) & (icmp eq (A & 15), 8) -> false.
   // (icmp ne (A & 6), 0) & (icmp eq (A & 15), 8) -> false.
   return ConstantInt::get(LHS->getType(), !IsAnd);
 }
 
 /// Try to fold (icmp(A & B) ==/!= 0) &/| (icmp(A & D) ==/!= E) into a single
 /// (icmp(A & X) ==/!= Y), where the left-hand side and the right hand side
 /// aren't of the common mask pattern type.
 static Value *foldLogOpOfMaskedICmpsAsymmetric(
     ICmpInst *LHS, ICmpInst *RHS, bool IsAnd, Value *A, Value *B, Value *C,
     Value *D, Value *E, ICmpInst::Predicate PredL, ICmpInst::Predicate PredR,
     unsigned LHSMask, unsigned RHSMask, InstCombiner::BuilderTy &Builder) {
   assert(ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) &&
          "Expected equality predicates for masked type of icmps.");
   // Handle Mask_NotAllZeros-BMask_Mixed cases.
   // (icmp ne/eq (A & B), C) &/| (icmp eq/ne (A & D), E), or
   // (icmp eq/ne (A & B), C) &/| (icmp ne/eq (A & D), E)
   //    which gets swapped to
   //    (icmp ne/eq (A & D), E) &/| (icmp eq/ne (A & B), C).
   if (!IsAnd) {
     LHSMask = conjugateICmpMask(LHSMask);
     RHSMask = conjugateICmpMask(RHSMask);
   }
   if ((LHSMask & Mask_NotAllZeros) && (RHSMask & BMask_Mixed)) {
     if (Value *V = foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed(
             LHS, RHS, IsAnd, A, B, C, D, E,
             PredL, PredR, Builder)) {
       return V;
     }
   } else if ((LHSMask & BMask_Mixed) && (RHSMask & Mask_NotAllZeros)) {
     if (Value *V = foldLogOpOfMaskedICmps_NotAllZeros_BMask_Mixed(
             RHS, LHS, IsAnd, A, D, E, B, C,
             PredR, PredL, Builder)) {
       return V;
     }
   }
   return nullptr;
 }
 
 /// Try to fold (icmp(A & B) ==/!= C) &/| (icmp(A & D) ==/!= E)
 /// into a single (icmp(A & X) ==/!= Y).
 static Value *foldLogOpOfMaskedICmps(ICmpInst *LHS, ICmpInst *RHS, bool IsAnd,
                                      InstCombiner::BuilderTy &Builder) {
   Value *A = nullptr, *B = nullptr, *C = nullptr, *D = nullptr, *E = nullptr;
   ICmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
   Optional<std::pair<unsigned, unsigned>> MaskPair =
       getMaskedTypeForICmpPair(A, B, C, D, E, LHS, RHS, PredL, PredR);
   if (!MaskPair)
     return nullptr;
   assert(ICmpInst::isEquality(PredL) && ICmpInst::isEquality(PredR) &&
          "Expected equality predicates for masked type of icmps.");
   unsigned LHSMask = MaskPair->first;
   unsigned RHSMask = MaskPair->second;
   unsigned Mask = LHSMask & RHSMask;
   if (Mask == 0) {
     // Even if the two sides don't share a common pattern, check if folding can
     // still happen.
     if (Value *V = foldLogOpOfMaskedICmpsAsymmetric(
             LHS, RHS, IsAnd, A, B, C, D, E, PredL, PredR, LHSMask, RHSMask,
             Builder))
       return V;
     return nullptr;
   }
 
   // In full generality:
   //     (icmp (A & B) Op C) | (icmp (A & D) Op E)
   // ==  ![ (icmp (A & B) !Op C) & (icmp (A & D) !Op E) ]
   //
   // If the latter can be converted into (icmp (A & X) Op Y) then the former is
   // equivalent to (icmp (A & X) !Op Y).
   //
   // Therefore, we can pretend for the rest of this function that we're dealing
   // with the conjunction, provided we flip the sense of any comparisons (both
   // input and output).
 
   // In most cases we're going to produce an EQ for the "&&" case.
   ICmpInst::Predicate NewCC = IsAnd ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
   if (!IsAnd) {
     // Convert the masking analysis into its equivalent with negated
     // comparisons.
     Mask = conjugateICmpMask(Mask);
   }
 
   if (Mask & Mask_AllZeros) {
     // (icmp eq (A & B), 0) & (icmp eq (A & D), 0)
     // -> (icmp eq (A & (B|D)), 0)
     Value *NewOr = Builder.CreateOr(B, D);
     Value *NewAnd = Builder.CreateAnd(A, NewOr);
     // We can't use C as zero because we might actually handle
     //   (icmp ne (A & B), B) & (icmp ne (A & D), D)
     // with B and D, having a single bit set.
     Value *Zero = Constant::getNullValue(A->getType());
     return Builder.CreateICmp(NewCC, NewAnd, Zero);
   }
   if (Mask & BMask_AllOnes) {
     // (icmp eq (A & B), B) & (icmp eq (A & D), D)
     // -> (icmp eq (A & (B|D)), (B|D))
     Value *NewOr = Builder.CreateOr(B, D);
     Value *NewAnd = Builder.CreateAnd(A, NewOr);
     return Builder.CreateICmp(NewCC, NewAnd, NewOr);
   }
   if (Mask & AMask_AllOnes) {
     // (icmp eq (A & B), A) & (icmp eq (A & D), A)
     // -> (icmp eq (A & (B&D)), A)
     Value *NewAnd1 = Builder.CreateAnd(B, D);
     Value *NewAnd2 = Builder.CreateAnd(A, NewAnd1);
     return Builder.CreateICmp(NewCC, NewAnd2, A);
   }
 
   // Remaining cases assume at least that B and D are constant, and depend on
   // their actual values. This isn't strictly necessary, just a "handle the
   // easy cases for now" decision.
   const APInt *ConstB, *ConstD;
   if (!match(B, m_APInt(ConstB)) || !match(D, m_APInt(ConstD)))
     return nullptr;
 
   if (Mask & (Mask_NotAllZeros | BMask_NotAllOnes)) {
     // (icmp ne (A & B), 0) & (icmp ne (A & D), 0) and
     // (icmp ne (A & B), B) & (icmp ne (A & D), D)
     //     -> (icmp ne (A & B), 0) or (icmp ne (A & D), 0)
     // Only valid if one of the masks is a superset of the other (check "B&D" is
     // the same as either B or D).
     APInt NewMask = *ConstB & *ConstD;
     if (NewMask == *ConstB)
       return LHS;
     else if (NewMask == *ConstD)
       return RHS;
   }
 
   if (Mask & AMask_NotAllOnes) {
     // (icmp ne (A & B), B) & (icmp ne (A & D), D)
     //     -> (icmp ne (A & B), A) or (icmp ne (A & D), A)
     // Only valid if one of the masks is a superset of the other (check "B|D" is
     // the same as either B or D).
     APInt NewMask = *ConstB | *ConstD;
     if (NewMask == *ConstB)
       return LHS;
     else if (NewMask == *ConstD)
       return RHS;
   }
 
   if (Mask & BMask_Mixed) {
     // (icmp eq (A & B), C) & (icmp eq (A & D), E)
     // We already know that B & C == C && D & E == E.
     // If we can prove that (B & D) & (C ^ E) == 0, that is, the bits of
     // C and E, which are shared by both the mask B and the mask D, don't
     // contradict, then we can transform to
     // -> (icmp eq (A & (B|D)), (C|E))
     // Currently, we only handle the case of B, C, D, and E being constant.
     // We can't simply use C and E because we might actually handle
     //   (icmp ne (A & B), B) & (icmp eq (A & D), D)
     // with B and D, having a single bit set.
     const APInt *OldConstC, *OldConstE;
     if (!match(C, m_APInt(OldConstC)) || !match(E, m_APInt(OldConstE)))
       return nullptr;
 
     const APInt ConstC = PredL != NewCC ? *ConstB ^ *OldConstC : *OldConstC;
     const APInt ConstE = PredR != NewCC ? *ConstD ^ *OldConstE : *OldConstE;
 
     // If there is a conflict, we should actually return a false for the
     // whole construct.
     if (((*ConstB & *ConstD) & (ConstC ^ ConstE)).getBoolValue())
       return ConstantInt::get(LHS->getType(), !IsAnd);
 
     Value *NewOr1 = Builder.CreateOr(B, D);
     Value *NewAnd = Builder.CreateAnd(A, NewOr1);
     Constant *NewOr2 = ConstantInt::get(A->getType(), ConstC | ConstE);
     return Builder.CreateICmp(NewCC, NewAnd, NewOr2);
   }
 
   return nullptr;
 }
 
 /// Try to fold a signed range checked with lower bound 0 to an unsigned icmp.
 /// Example: (icmp sge x, 0) & (icmp slt x, n) --> icmp ult x, n
 /// If \p Inverted is true then the check is for the inverted range, e.g.
 /// (icmp slt x, 0) | (icmp sgt x, n) --> icmp ugt x, n
 Value *InstCombinerImpl::simplifyRangeCheck(ICmpInst *Cmp0, ICmpInst *Cmp1,
                                             bool Inverted) {
   // Check the lower range comparison, e.g. x >= 0
   // InstCombine already ensured that if there is a constant it's on the RHS.
   ConstantInt *RangeStart = dyn_cast<ConstantInt>(Cmp0->getOperand(1));
   if (!RangeStart)
     return nullptr;
 
   ICmpInst::Predicate Pred0 = (Inverted ? Cmp0->getInversePredicate() :
                                Cmp0->getPredicate());
 
   // Accept x > -1 or x >= 0 (after potentially inverting the predicate).
   if (!((Pred0 == ICmpInst::ICMP_SGT && RangeStart->isMinusOne()) ||
         (Pred0 == ICmpInst::ICMP_SGE && RangeStart->isZero())))
     return nullptr;
 
   ICmpInst::Predicate Pred1 = (Inverted ? Cmp1->getInversePredicate() :
                                Cmp1->getPredicate());
 
   Value *Input = Cmp0->getOperand(0);
   Value *RangeEnd;
   if (Cmp1->getOperand(0) == Input) {
     // For the upper range compare we have: icmp x, n
     RangeEnd = Cmp1->getOperand(1);
   } else if (Cmp1->getOperand(1) == Input) {
     // For the upper range compare we have: icmp n, x
     RangeEnd = Cmp1->getOperand(0);
     Pred1 = ICmpInst::getSwappedPredicate(Pred1);
   } else {
     return nullptr;
   }
 
   // Check the upper range comparison, e.g. x < n
   ICmpInst::Predicate NewPred;
   switch (Pred1) {
     case ICmpInst::ICMP_SLT: NewPred = ICmpInst::ICMP_ULT; break;
     case ICmpInst::ICMP_SLE: NewPred = ICmpInst::ICMP_ULE; break;
     default: return nullptr;
   }
 
   // This simplification is only valid if the upper range is not negative.
   KnownBits Known = computeKnownBits(RangeEnd, /*Depth=*/0, Cmp1);
   if (!Known.isNonNegative())
     return nullptr;
 
   if (Inverted)
     NewPred = ICmpInst::getInversePredicate(NewPred);
 
   return Builder.CreateICmp(NewPred, Input, RangeEnd);
 }
 
 static Value *
 foldAndOrOfEqualityCmpsWithConstants(ICmpInst *LHS, ICmpInst *RHS,
                                      bool JoinedByAnd,
                                      InstCombiner::BuilderTy &Builder) {
   Value *X = LHS->getOperand(0);
   if (X != RHS->getOperand(0))
     return nullptr;
 
   const APInt *C1, *C2;
   if (!match(LHS->getOperand(1), m_APInt(C1)) ||
       !match(RHS->getOperand(1), m_APInt(C2)))
     return nullptr;
 
   // We only handle (X != C1 && X != C2) and (X == C1 || X == C2).
   ICmpInst::Predicate Pred = LHS->getPredicate();
   if (Pred !=  RHS->getPredicate())
     return nullptr;
   if (JoinedByAnd && Pred != ICmpInst::ICMP_NE)
     return nullptr;
   if (!JoinedByAnd && Pred != ICmpInst::ICMP_EQ)
     return nullptr;
 
   // The larger unsigned constant goes on the right.
   if (C1->ugt(*C2))
     std::swap(C1, C2);
 
   APInt Xor = *C1 ^ *C2;
   if (Xor.isPowerOf2()) {
     // If LHSC and RHSC differ by only one bit, then set that bit in X and
     // compare against the larger constant:
     // (X == C1 || X == C2) --> (X | (C1 ^ C2)) == C2
     // (X != C1 && X != C2) --> (X | (C1 ^ C2)) != C2
     // We choose an 'or' with a Pow2 constant rather than the inverse mask with
     // 'and' because that may lead to smaller codegen from a smaller constant.
     Value *Or = Builder.CreateOr(X, ConstantInt::get(X->getType(), Xor));
     return Builder.CreateICmp(Pred, Or, ConstantInt::get(X->getType(), *C2));
   }
 
   return nullptr;
 }
 
 // Fold (iszero(A & K1) | iszero(A & K2)) -> (A & (K1 | K2)) != (K1 | K2)
 // Fold (!iszero(A & K1) & !iszero(A & K2)) -> (A & (K1 | K2)) == (K1 | K2)
 Value *InstCombinerImpl::foldAndOrOfICmpsOfAndWithPow2(ICmpInst *LHS,
                                                        ICmpInst *RHS,
                                                        Instruction *CxtI,
                                                        bool IsAnd,
                                                        bool IsLogical) {
   CmpInst::Predicate Pred = IsAnd ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ;
   if (LHS->getPredicate() != Pred || RHS->getPredicate() != Pred)
     return nullptr;
 
   if (!match(LHS->getOperand(1), m_Zero()) ||
       !match(RHS->getOperand(1), m_Zero()))
     return nullptr;
 
   Value *L1, *L2, *R1, *R2;
   if (match(LHS->getOperand(0), m_And(m_Value(L1), m_Value(L2))) &&
       match(RHS->getOperand(0), m_And(m_Value(R1), m_Value(R2)))) {
     if (L1 == R2 || L2 == R2)
       std::swap(R1, R2);
     if (L2 == R1)
       std::swap(L1, L2);
 
     if (L1 == R1 &&
         isKnownToBeAPowerOfTwo(L2, false, 0, CxtI) &&
         isKnownToBeAPowerOfTwo(R2, false, 0, CxtI)) {
       // If this is a logical and/or, then we must prevent propagation of a
       // poison value from the RHS by inserting freeze.
       if (IsLogical)
         R2 = Builder.CreateFreeze(R2);
       Value *Mask = Builder.CreateOr(L2, R2);
       Value *Masked = Builder.CreateAnd(L1, Mask);
       auto NewPred = IsAnd ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
       return Builder.CreateICmp(NewPred, Masked, Mask);
     }
   }
 
   return nullptr;
 }
 
 /// General pattern:
 ///   X & Y
 ///
 /// Where Y is checking that all the high bits (covered by a mask 4294967168)
 /// are uniform, i.e.  %arg & 4294967168  can be either  4294967168  or  0
 /// Pattern can be one of:
 ///   %t = add        i32 %arg,    128
 ///   %r = icmp   ult i32 %t,      256
 /// Or
 ///   %t0 = shl       i32 %arg,    24
 ///   %t1 = ashr      i32 %t0,     24
 ///   %r  = icmp  eq  i32 %t1,     %arg
 /// Or
 ///   %t0 = trunc     i32 %arg  to i8
 ///   %t1 = sext      i8  %t0   to i32
 ///   %r  = icmp  eq  i32 %t1,     %arg
 /// This pattern is a signed truncation check.
 ///
 /// And X is checking that some bit in that same mask is zero.
 /// I.e. can be one of:
 ///   %r = icmp sgt i32   %arg,    -1
 /// Or
 ///   %t = and      i32   %arg,    2147483648
 ///   %r = icmp eq  i32   %t,      0
 ///
 /// Since we are checking that all the bits in that mask are the same,
 /// and a particular bit is zero, what we are really checking is that all the
 /// masked bits are zero.
 /// So this should be transformed to:
 ///   %r = icmp ult i32 %arg, 128
 static Value *foldSignedTruncationCheck(ICmpInst *ICmp0, ICmpInst *ICmp1,
                                         Instruction &CxtI,
                                         InstCombiner::BuilderTy &Builder) {
   assert(CxtI.getOpcode() == Instruction::And);
 
   // Match  icmp ult (add %arg, C01), C1   (C1 == C01 << 1; powers of two)
   auto tryToMatchSignedTruncationCheck = [](ICmpInst *ICmp, Value *&X,
                                             APInt &SignBitMask) -> bool {
     CmpInst::Predicate Pred;
     const APInt *I01, *I1; // powers of two; I1 == I01 << 1
     if (!(match(ICmp,
                 m_ICmp(Pred, m_Add(m_Value(X), m_Power2(I01)), m_Power2(I1))) &&
           Pred == ICmpInst::ICMP_ULT && I1->ugt(*I01) && I01->shl(1) == *I1))
       return false;
     // Which bit is the new sign bit as per the 'signed truncation' pattern?
     SignBitMask = *I01;
     return true;
   };
 
   // One icmp needs to be 'signed truncation check'.
   // We need to match this first, else we will mismatch commutative cases.
   Value *X1;
   APInt HighestBit;
   ICmpInst *OtherICmp;
   if (tryToMatchSignedTruncationCheck(ICmp1, X1, HighestBit))
     OtherICmp = ICmp0;
   else if (tryToMatchSignedTruncationCheck(ICmp0, X1, HighestBit))
     OtherICmp = ICmp1;
   else
     return nullptr;
 
   assert(HighestBit.isPowerOf2() && "expected to be power of two (non-zero)");
 
   // Try to match/decompose into:  icmp eq (X & Mask), 0
   auto tryToDecompose = [](ICmpInst *ICmp, Value *&X,
                            APInt &UnsetBitsMask) -> bool {
     CmpInst::Predicate Pred = ICmp->getPredicate();
     // Can it be decomposed into  icmp eq (X & Mask), 0  ?
     if (llvm::decomposeBitTestICmp(ICmp->getOperand(0), ICmp->getOperand(1),
                                    Pred, X, UnsetBitsMask,
                                    /*LookThroughTrunc=*/false) &&
         Pred == ICmpInst::ICMP_EQ)
       return true;
     // Is it  icmp eq (X & Mask), 0  already?
     const APInt *Mask;
     if (match(ICmp, m_ICmp(Pred, m_And(m_Value(X), m_APInt(Mask)), m_Zero())) &&
         Pred == ICmpInst::ICMP_EQ) {
       UnsetBitsMask = *Mask;
       return true;
     }
     return false;
   };
 
   // And the other icmp needs to be decomposable into a bit test.
   Value *X0;
   APInt UnsetBitsMask;
   if (!tryToDecompose(OtherICmp, X0, UnsetBitsMask))
     return nullptr;
 
   assert(!UnsetBitsMask.isZero() && "empty mask makes no sense.");
 
   // Are they working on the same value?
   Value *X;
   if (X1 == X0) {
     // Ok as is.
     X = X1;
   } else if (match(X0, m_Trunc(m_Specific(X1)))) {
     UnsetBitsMask = UnsetBitsMask.zext(X1->getType()->getScalarSizeInBits());
     X = X1;
   } else
     return nullptr;
 
   // So which bits should be uniform as per the 'signed truncation check'?
   // (all the bits starting with (i.e. including) HighestBit)
   APInt SignBitsMask = ~(HighestBit - 1U);
 
   // UnsetBitsMask must have some common bits with SignBitsMask,
   if (!UnsetBitsMask.intersects(SignBitsMask))
     return nullptr;
 
   // Does UnsetBitsMask contain any bits outside of SignBitsMask?
   if (!UnsetBitsMask.isSubsetOf(SignBitsMask)) {
     APInt OtherHighestBit = (~UnsetBitsMask) + 1U;
     if (!OtherHighestBit.isPowerOf2())
       return nullptr;
     HighestBit = APIntOps::umin(HighestBit, OtherHighestBit);
   }
   // Else, if it does not, then all is ok as-is.
 
   // %r = icmp ult %X, SignBit
   return Builder.CreateICmpULT(X, ConstantInt::get(X->getType(), HighestBit),
                                CxtI.getName() + ".simplified");
 }
 
 /// Reduce a pair of compares that check if a value has exactly 1 bit set.
 static Value *foldIsPowerOf2(ICmpInst *Cmp0, ICmpInst *Cmp1, bool JoinedByAnd,
                              InstCombiner::BuilderTy &Builder) {
   // Handle 'and' / 'or' commutation: make the equality check the first operand.
   if (JoinedByAnd && Cmp1->getPredicate() == ICmpInst::ICMP_NE)
     std::swap(Cmp0, Cmp1);
   else if (!JoinedByAnd && Cmp1->getPredicate() == ICmpInst::ICMP_EQ)
     std::swap(Cmp0, Cmp1);
 
   // (X != 0) && (ctpop(X) u< 2) --> ctpop(X) == 1
   CmpInst::Predicate Pred0, Pred1;
   Value *X;
   if (JoinedByAnd && match(Cmp0, m_ICmp(Pred0, m_Value(X), m_ZeroInt())) &&
       match(Cmp1, m_ICmp(Pred1, m_Intrinsic<Intrinsic::ctpop>(m_Specific(X)),
                          m_SpecificInt(2))) &&
       Pred0 == ICmpInst::ICMP_NE && Pred1 == ICmpInst::ICMP_ULT) {
     Value *CtPop = Cmp1->getOperand(0);
     return Builder.CreateICmpEQ(CtPop, ConstantInt::get(CtPop->getType(), 1));
   }
   // (X == 0) || (ctpop(X) u> 1) --> ctpop(X) != 1
   if (!JoinedByAnd && match(Cmp0, m_ICmp(Pred0, m_Value(X), m_ZeroInt())) &&
       match(Cmp1, m_ICmp(Pred1, m_Intrinsic<Intrinsic::ctpop>(m_Specific(X)),
                          m_SpecificInt(1))) &&
       Pred0 == ICmpInst::ICMP_EQ && Pred1 == ICmpInst::ICMP_UGT) {
     Value *CtPop = Cmp1->getOperand(0);
     return Builder.CreateICmpNE(CtPop, ConstantInt::get(CtPop->getType(), 1));
   }
   return nullptr;
 }
 
 /// Commuted variants are assumed to be handled by calling this function again
 /// with the parameters swapped.
 static Value *foldUnsignedUnderflowCheck(ICmpInst *ZeroICmp,
                                          ICmpInst *UnsignedICmp, bool IsAnd,
                                          const SimplifyQuery &Q,
                                          InstCombiner::BuilderTy &Builder) {
   Value *ZeroCmpOp;
   ICmpInst::Predicate EqPred;
   if (!match(ZeroICmp, m_ICmp(EqPred, m_Value(ZeroCmpOp), m_Zero())) ||
       !ICmpInst::isEquality(EqPred))
     return nullptr;
 
   auto IsKnownNonZero = [&](Value *V) {
     return isKnownNonZero(V, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT);
   };
 
   ICmpInst::Predicate UnsignedPred;
 
   Value *A, *B;
   if (match(UnsignedICmp,
             m_c_ICmp(UnsignedPred, m_Specific(ZeroCmpOp), m_Value(A))) &&
       match(ZeroCmpOp, m_c_Add(m_Specific(A), m_Value(B))) &&
       (ZeroICmp->hasOneUse() || UnsignedICmp->hasOneUse())) {
     auto GetKnownNonZeroAndOther = [&](Value *&NonZero, Value *&Other) {
       if (!IsKnownNonZero(NonZero))
         std::swap(NonZero, Other);
       return IsKnownNonZero(NonZero);
     };
 
     // Given  ZeroCmpOp = (A + B)
     //   ZeroCmpOp <= A && ZeroCmpOp != 0  -->  (0-B) <  A
     //   ZeroCmpOp >  A || ZeroCmpOp == 0  -->  (0-B) >= A
     //
     //   ZeroCmpOp <  A && ZeroCmpOp != 0  -->  (0-X) <  Y  iff
     //   ZeroCmpOp >= A || ZeroCmpOp == 0  -->  (0-X) >= Y  iff
     //     with X being the value (A/B) that is known to be non-zero,
     //     and Y being remaining value.
     if (UnsignedPred == ICmpInst::ICMP_ULE && EqPred == ICmpInst::ICMP_NE &&
         IsAnd)
       return Builder.CreateICmpULT(Builder.CreateNeg(B), A);
     if (UnsignedPred == ICmpInst::ICMP_ULT && EqPred == ICmpInst::ICMP_NE &&
         IsAnd && GetKnownNonZeroAndOther(B, A))
       return Builder.CreateICmpULT(Builder.CreateNeg(B), A);
     if (UnsignedPred == ICmpInst::ICMP_UGT && EqPred == ICmpInst::ICMP_EQ &&
         !IsAnd)
       return Builder.CreateICmpUGE(Builder.CreateNeg(B), A);
     if (UnsignedPred == ICmpInst::ICMP_UGE && EqPred == ICmpInst::ICMP_EQ &&
         !IsAnd && GetKnownNonZeroAndOther(B, A))
       return Builder.CreateICmpUGE(Builder.CreateNeg(B), A);
   }
 
   Value *Base, *Offset;
   if (!match(ZeroCmpOp, m_Sub(m_Value(Base), m_Value(Offset))))
     return nullptr;
 
   if (!match(UnsignedICmp,
              m_c_ICmp(UnsignedPred, m_Specific(Base), m_Specific(Offset))) ||
       !ICmpInst::isUnsigned(UnsignedPred))
     return nullptr;
 
   // Base >=/> Offset && (Base - Offset) != 0  <-->  Base > Offset
   // (no overflow and not null)
   if ((UnsignedPred == ICmpInst::ICMP_UGE ||
        UnsignedPred == ICmpInst::ICMP_UGT) &&
       EqPred == ICmpInst::ICMP_NE && IsAnd)
     return Builder.CreateICmpUGT(Base, Offset);
 
   // Base <=/< Offset || (Base - Offset) == 0  <-->  Base <= Offset
   // (overflow or null)
   if ((UnsignedPred == ICmpInst::ICMP_ULE ||
        UnsignedPred == ICmpInst::ICMP_ULT) &&
       EqPred == ICmpInst::ICMP_EQ && !IsAnd)
     return Builder.CreateICmpULE(Base, Offset);
 
   // Base <= Offset && (Base - Offset) != 0  -->  Base < Offset
   if (UnsignedPred == ICmpInst::ICMP_ULE && EqPred == ICmpInst::ICMP_NE &&
       IsAnd)
     return Builder.CreateICmpULT(Base, Offset);
 
   // Base > Offset || (Base - Offset) == 0  -->  Base >= Offset
   if (UnsignedPred == ICmpInst::ICMP_UGT && EqPred == ICmpInst::ICMP_EQ &&
       !IsAnd)
     return Builder.CreateICmpUGE(Base, Offset);
 
   return nullptr;
 }
 
 struct IntPart {
   Value *From;
   unsigned StartBit;
   unsigned NumBits;
 };
 
 /// Match an extraction of bits from an integer.
 static Optional<IntPart> matchIntPart(Value *V) {
   Value *X;
   if (!match(V, m_OneUse(m_Trunc(m_Value(X)))))
     return None;
 
   unsigned NumOriginalBits = X->getType()->getScalarSizeInBits();
   unsigned NumExtractedBits = V->getType()->getScalarSizeInBits();
   Value *Y;
   const APInt *Shift;
   // For a trunc(lshr Y, Shift) pattern, make sure we're only extracting bits
   // from Y, not any shifted-in zeroes.
   if (match(X, m_OneUse(m_LShr(m_Value(Y), m_APInt(Shift)))) &&
       Shift->ule(NumOriginalBits - NumExtractedBits))
     return {{Y, (unsigned)Shift->getZExtValue(), NumExtractedBits}};
   return {{X, 0, NumExtractedBits}};
 }
 
 /// Materialize an extraction of bits from an integer in IR.
 static Value *extractIntPart(const IntPart &P, IRBuilderBase &Builder) {
   Value *V = P.From;
   if (P.StartBit)
     V = Builder.CreateLShr(V, P.StartBit);
   Type *TruncTy = V->getType()->getWithNewBitWidth(P.NumBits);
   if (TruncTy != V->getType())
     V = Builder.CreateTrunc(V, TruncTy);
   return V;
 }
 
 /// (icmp eq X0, Y0) & (icmp eq X1, Y1) -> icmp eq X01, Y01
 /// (icmp ne X0, Y0) | (icmp ne X1, Y1) -> icmp ne X01, Y01
 /// where X0, X1 and Y0, Y1 are adjacent parts extracted from an integer.
 Value *InstCombinerImpl::foldEqOfParts(ICmpInst *Cmp0, ICmpInst *Cmp1,
                                        bool IsAnd) {
   if (!Cmp0->hasOneUse() || !Cmp1->hasOneUse())
     return nullptr;
 
   CmpInst::Predicate Pred = IsAnd ? CmpInst::ICMP_EQ : CmpInst::ICMP_NE;
   if (Cmp0->getPredicate() != Pred || Cmp1->getPredicate() != Pred)
     return nullptr;
 
   Optional<IntPart> L0 = matchIntPart(Cmp0->getOperand(0));
   Optional<IntPart> R0 = matchIntPart(Cmp0->getOperand(1));
   Optional<IntPart> L1 = matchIntPart(Cmp1->getOperand(0));
   Optional<IntPart> R1 = matchIntPart(Cmp1->getOperand(1));
   if (!L0 || !R0 || !L1 || !R1)
     return nullptr;
 
   // Make sure the LHS/RHS compare a part of the same value, possibly after
   // an operand swap.
   if (L0->From != L1->From || R0->From != R1->From) {
     if (L0->From != R1->From || R0->From != L1->From)
       return nullptr;
     std::swap(L1, R1);
   }
 
   // Make sure the extracted parts are adjacent, canonicalizing to L0/R0 being
   // the low part and L1/R1 being the high part.
   if (L0->StartBit + L0->NumBits != L1->StartBit ||
       R0->StartBit + R0->NumBits != R1->StartBit) {
     if (L1->StartBit + L1->NumBits != L0->StartBit ||
         R1->StartBit + R1->NumBits != R0->StartBit)
       return nullptr;
     std::swap(L0, L1);
     std::swap(R0, R1);
   }
 
   // We can simplify to a comparison of these larger parts of the integers.
   IntPart L = {L0->From, L0->StartBit, L0->NumBits + L1->NumBits};
   IntPart R = {R0->From, R0->StartBit, R0->NumBits + R1->NumBits};
   Value *LValue = extractIntPart(L, Builder);
   Value *RValue = extractIntPart(R, Builder);
   return Builder.CreateICmp(Pred, LValue, RValue);
 }
 
 /// Reduce logic-of-compares with equality to a constant by substituting a
 /// common operand with the constant. Callers are expected to call this with
 /// Cmp0/Cmp1 switched to handle logic op commutativity.
 static Value *foldAndOrOfICmpsWithConstEq(ICmpInst *Cmp0, ICmpInst *Cmp1,
                                           BinaryOperator &Logic,
                                           InstCombiner::BuilderTy &Builder,
                                           const SimplifyQuery &Q) {
   bool IsAnd = Logic.getOpcode() == Instruction::And;
   assert((IsAnd || Logic.getOpcode() == Instruction::Or) && "Wrong logic op");
 
   // Match an equality compare with a non-poison constant as Cmp0.
   // Also, give up if the compare can be constant-folded to avoid looping.
   ICmpInst::Predicate Pred0;
   Value *X;
   Constant *C;
   if (!match(Cmp0, m_ICmp(Pred0, m_Value(X), m_Constant(C))) ||
       !isGuaranteedNotToBeUndefOrPoison(C) || isa<Constant>(X))
     return nullptr;
   if ((IsAnd && Pred0 != ICmpInst::ICMP_EQ) ||
       (!IsAnd && Pred0 != ICmpInst::ICMP_NE))
     return nullptr;
 
   // The other compare must include a common operand (X). Canonicalize the
   // common operand as operand 1 (Pred1 is swapped if the common operand was
   // operand 0).
   Value *Y;
   ICmpInst::Predicate Pred1;
   if (!match(Cmp1, m_c_ICmp(Pred1, m_Value(Y), m_Deferred(X))))
     return nullptr;
 
   // Replace variable with constant value equivalence to remove a variable use:
   // (X == C) && (Y Pred1 X) --> (X == C) && (Y Pred1 C)
   // (X != C) || (Y Pred1 X) --> (X != C) || (Y Pred1 C)
   // Can think of the 'or' substitution with the 'and' bool equivalent:
   // A || B --> A || (!A && B)
   Value *SubstituteCmp = SimplifyICmpInst(Pred1, Y, C, Q);
   if (!SubstituteCmp) {
     // If we need to create a new instruction, require that the old compare can
     // be removed.
     if (!Cmp1->hasOneUse())
       return nullptr;
     SubstituteCmp = Builder.CreateICmp(Pred1, Y, C);
   }
   return Builder.CreateBinOp(Logic.getOpcode(), Cmp0, SubstituteCmp);
 }
 
 /// Fold (icmp Pred1 V1, C1) & (icmp Pred2 V2, C2)
 /// or   (icmp Pred1 V1, C1) | (icmp Pred2 V2, C2)
 /// into a single comparison using range-based reasoning.
 static Value *foldAndOrOfICmpsUsingRanges(
     ICmpInst::Predicate Pred1, Value *V1, const APInt &C1,
     ICmpInst::Predicate Pred2, Value *V2, const APInt &C2,
     IRBuilderBase &Builder, bool IsAnd) {
   // Look through add of a constant offset on V1, V2, or both operands. This
   // allows us to interpret the V + C' < C'' range idiom into a proper range.
   const APInt *Offset1 = nullptr, *Offset2 = nullptr;
   if (V1 != V2) {
     Value *X;
     if (match(V1, m_Add(m_Value(X), m_APInt(Offset1))))
       V1 = X;
     if (match(V2, m_Add(m_Value(X), m_APInt(Offset2))))
       V2 = X;
   }
 
   if (V1 != V2)
     return nullptr;
 
   ConstantRange CR1 = ConstantRange::makeExactICmpRegion(Pred1, C1);
   if (Offset1)
     CR1 = CR1.subtract(*Offset1);
 
   ConstantRange CR2 = ConstantRange::makeExactICmpRegion(Pred2, C2);
   if (Offset2)
     CR2 = CR2.subtract(*Offset2);
 
   Optional<ConstantRange> CR =
       IsAnd ? CR1.exactIntersectWith(CR2) : CR1.exactUnionWith(CR2);
   if (!CR)
     return nullptr;
 
   CmpInst::Predicate NewPred;
   APInt NewC, Offset;
   CR->getEquivalentICmp(NewPred, NewC, Offset);
 
   Type *Ty = V1->getType();
   Value *NewV = V1;
   if (Offset != 0)
     NewV = Builder.CreateAdd(NewV, ConstantInt::get(Ty, Offset));
   return Builder.CreateICmp(NewPred, NewV, ConstantInt::get(Ty, NewC));
 }
 
 /// Fold (icmp)&(icmp) if possible.
 Value *InstCombinerImpl::foldAndOfICmps(ICmpInst *LHS, ICmpInst *RHS,
                                         BinaryOperator &And) {
   const SimplifyQuery Q = SQ.getWithInstruction(&And);
 
   // Fold (!iszero(A & K1) & !iszero(A & K2)) ->  (A & (K1 | K2)) == (K1 | K2)
   // if K1 and K2 are a one-bit mask.
   if (Value *V = foldAndOrOfICmpsOfAndWithPow2(LHS, RHS, &And,
                                                /* IsAnd */ true))
     return V;
 
   ICmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
 
   // (icmp1 A, B) & (icmp2 A, B) --> (icmp3 A, B)
   if (predicatesFoldable(PredL, PredR)) {
     if (LHS->getOperand(0) == RHS->getOperand(1) &&
         LHS->getOperand(1) == RHS->getOperand(0))
       LHS->swapOperands();
     if (LHS->getOperand(0) == RHS->getOperand(0) &&
         LHS->getOperand(1) == RHS->getOperand(1)) {
       Value *Op0 = LHS->getOperand(0), *Op1 = LHS->getOperand(1);
       unsigned Code = getICmpCode(LHS) & getICmpCode(RHS);
       bool IsSigned = LHS->isSigned() || RHS->isSigned();
       return getNewICmpValue(Code, IsSigned, Op0, Op1, Builder);
     }
   }
 
   // handle (roughly):  (icmp eq (A & B), C) & (icmp eq (A & D), E)
   if (Value *V = foldLogOpOfMaskedICmps(LHS, RHS, true, Builder))
     return V;
 
   if (Value *V = foldAndOrOfICmpsWithConstEq(LHS, RHS, And, Builder, Q))
     return V;
   if (Value *V = foldAndOrOfICmpsWithConstEq(RHS, LHS, And, Builder, Q))
     return V;
 
   // E.g. (icmp sge x, 0) & (icmp slt x, n) --> icmp ult x, n
   if (Value *V = simplifyRangeCheck(LHS, RHS, /*Inverted=*/false))
     return V;
 
   // E.g. (icmp slt x, n) & (icmp sge x, 0) --> icmp ult x, n
   if (Value *V = simplifyRangeCheck(RHS, LHS, /*Inverted=*/false))
     return V;
 
   if (Value *V = foldAndOrOfEqualityCmpsWithConstants(LHS, RHS, true, Builder))
     return V;
 
   if (Value *V = foldSignedTruncationCheck(LHS, RHS, And, Builder))
     return V;
 
   if (Value *V = foldIsPowerOf2(LHS, RHS, true /* JoinedByAnd */, Builder))
     return V;
 
   if (Value *X =
           foldUnsignedUnderflowCheck(LHS, RHS, /*IsAnd=*/true, Q, Builder))
     return X;
   if (Value *X =
           foldUnsignedUnderflowCheck(RHS, LHS, /*IsAnd=*/true, Q, Builder))
     return X;
 
   if (Value *X = foldEqOfParts(LHS, RHS, /*IsAnd=*/true))
     return X;
 
   // This only handles icmp of constants: (icmp1 A, C1) & (icmp2 B, C2).
   Value *LHS0 = LHS->getOperand(0), *RHS0 = RHS->getOperand(0);
 
   // (icmp eq A, 0) & (icmp eq B, 0) --> (icmp eq (A|B), 0)
   // TODO: Remove this when foldLogOpOfMaskedICmps can handle undefs.
   if (PredL == ICmpInst::ICMP_EQ && match(LHS->getOperand(1), m_ZeroInt()) &&
       PredR == ICmpInst::ICMP_EQ && match(RHS->getOperand(1), m_ZeroInt()) &&
       LHS0->getType() == RHS0->getType()) {
     Value *NewOr = Builder.CreateOr(LHS0, RHS0);
     return Builder.CreateICmp(PredL, NewOr,
                               Constant::getNullValue(NewOr->getType()));
   }
 
   const APInt *LHSC, *RHSC;
   if (!match(LHS->getOperand(1), m_APInt(LHSC)) ||
       !match(RHS->getOperand(1), m_APInt(RHSC)))
     return nullptr;
 
   // (trunc x) == C1 & (and x, CA) == C2 -> (and x, CA|CMAX) == C1|C2
   // where CMAX is the all ones value for the truncated type,
   // iff the lower bits of C2 and CA are zero.
   if (PredL == ICmpInst::ICMP_EQ && PredL == PredR && LHS->hasOneUse() &&
       RHS->hasOneUse()) {
     Value *V;
     const APInt *AndC, *SmallC = nullptr, *BigC = nullptr;
 
     // (trunc x) == C1 & (and x, CA) == C2
     // (and x, CA) == C2 & (trunc x) == C1
     if (match(RHS0, m_Trunc(m_Value(V))) &&
         match(LHS0, m_And(m_Specific(V), m_APInt(AndC)))) {
       SmallC = RHSC;
       BigC = LHSC;
     } else if (match(LHS0, m_Trunc(m_Value(V))) &&
                match(RHS0, m_And(m_Specific(V), m_APInt(AndC)))) {
       SmallC = LHSC;
       BigC = RHSC;
     }
 
     if (SmallC && BigC) {
       unsigned BigBitSize = BigC->getBitWidth();
       unsigned SmallBitSize = SmallC->getBitWidth();
 
       // Check that the low bits are zero.
       APInt Low = APInt::getLowBitsSet(BigBitSize, SmallBitSize);
       if ((Low & *AndC).isZero() && (Low & *BigC).isZero()) {
         Value *NewAnd = Builder.CreateAnd(V, Low | *AndC);
         APInt N = SmallC->zext(BigBitSize) | *BigC;
         Value *NewVal = ConstantInt::get(NewAnd->getType(), N);
         return Builder.CreateICmp(PredL, NewAnd, NewVal);
       }
     }
   }
 
   return foldAndOrOfICmpsUsingRanges(PredL, LHS0, *LHSC, PredR, RHS0, *RHSC,
                                      Builder, /* IsAnd */ true);
 }
 
 Value *InstCombinerImpl::foldLogicOfFCmps(FCmpInst *LHS, FCmpInst *RHS,
                                           bool IsAnd) {
   Value *LHS0 = LHS->getOperand(0), *LHS1 = LHS->getOperand(1);
   Value *RHS0 = RHS->getOperand(0), *RHS1 = RHS->getOperand(1);
   FCmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
 
   if (LHS0 == RHS1 && RHS0 == LHS1) {
     // Swap RHS operands to match LHS.
     PredR = FCmpInst::getSwappedPredicate(PredR);
     std::swap(RHS0, RHS1);
   }
 
   // Simplify (fcmp cc0 x, y) & (fcmp cc1 x, y).
   // Suppose the relation between x and y is R, where R is one of
   // U(1000), L(0100), G(0010) or E(0001), and CC0 and CC1 are the bitmasks for
   // testing the desired relations.
   //
   // Since (R & CC0) and (R & CC1) are either R or 0, we actually have this:
   //    bool(R & CC0) && bool(R & CC1)
   //  = bool((R & CC0) & (R & CC1))
   //  = bool(R & (CC0 & CC1)) <= by re-association, commutation, and idempotency
   //
   // Since (R & CC0) and (R & CC1) are either R or 0, we actually have this:
   //    bool(R & CC0) || bool(R & CC1)
   //  = bool((R & CC0) | (R & CC1))
   //  = bool(R & (CC0 | CC1)) <= by reversed distribution (contribution? ;)
   if (LHS0 == RHS0 && LHS1 == RHS1) {
     unsigned FCmpCodeL = getFCmpCode(PredL);
     unsigned FCmpCodeR = getFCmpCode(PredR);
     unsigned NewPred = IsAnd ? FCmpCodeL & FCmpCodeR : FCmpCodeL | FCmpCodeR;
     return getFCmpValue(NewPred, LHS0, LHS1, Builder);
   }
 
   if ((PredL == FCmpInst::FCMP_ORD && PredR == FCmpInst::FCMP_ORD && IsAnd) ||
       (PredL == FCmpInst::FCMP_UNO && PredR == FCmpInst::FCMP_UNO && !IsAnd)) {
     if (LHS0->getType() != RHS0->getType())
       return nullptr;
 
     // FCmp canonicalization ensures that (fcmp ord/uno X, X) and
     // (fcmp ord/uno X, C) will be transformed to (fcmp X, +0.0).
     if (match(LHS1, m_PosZeroFP()) && match(RHS1, m_PosZeroFP()))
       // Ignore the constants because they are obviously not NANs:
       // (fcmp ord x, 0.0) & (fcmp ord y, 0.0)  -> (fcmp ord x, y)
       // (fcmp uno x, 0.0) | (fcmp uno y, 0.0)  -> (fcmp uno x, y)
       return Builder.CreateFCmp(PredL, LHS0, RHS0);
   }
 
   return nullptr;
 }
 
 /// This a limited reassociation for a special case (see above) where we are
 /// checking if two values are either both NAN (unordered) or not-NAN (ordered).
 /// This could be handled more generally in '-reassociation', but it seems like
 /// an unlikely pattern for a large number of logic ops and fcmps.
 static Instruction *reassociateFCmps(BinaryOperator &BO,
                                      InstCombiner::BuilderTy &Builder) {
   Instruction::BinaryOps Opcode = BO.getOpcode();
   assert((Opcode == Instruction::And || Opcode == Instruction::Or) &&
          "Expecting and/or op for fcmp transform");
 
   // There are 4 commuted variants of the pattern. Canonicalize operands of this
   // logic op so an fcmp is operand 0 and a matching logic op is operand 1.
   Value *Op0 = BO.getOperand(0), *Op1 = BO.getOperand(1), *X;
   FCmpInst::Predicate Pred;
   if (match(Op1, m_FCmp(Pred, m_Value(), m_AnyZeroFP())))
     std::swap(Op0, Op1);
 
   // Match inner binop and the predicate for combining 2 NAN checks into 1.
   Value *BO10, *BO11;
   FCmpInst::Predicate NanPred = Opcode == Instruction::And ? FCmpInst::FCMP_ORD
                                                            : FCmpInst::FCMP_UNO;
   if (!match(Op0, m_FCmp(Pred, m_Value(X), m_AnyZeroFP())) || Pred != NanPred ||
       !match(Op1, m_BinOp(Opcode, m_Value(BO10), m_Value(BO11))))
     return nullptr;
 
   // The inner logic op must have a matching fcmp operand.
   Value *Y;
   if (!match(BO10, m_FCmp(Pred, m_Value(Y), m_AnyZeroFP())) ||
       Pred != NanPred || X->getType() != Y->getType())
     std::swap(BO10, BO11);
 
   if (!match(BO10, m_FCmp(Pred, m_Value(Y), m_AnyZeroFP())) ||
       Pred != NanPred || X->getType() != Y->getType())
     return nullptr;
 
   // and (fcmp ord X, 0), (and (fcmp ord Y, 0), Z) --> and (fcmp ord X, Y), Z
   // or  (fcmp uno X, 0), (or  (fcmp uno Y, 0), Z) --> or  (fcmp uno X, Y), Z
   Value *NewFCmp = Builder.CreateFCmp(Pred, X, Y);
   if (auto *NewFCmpInst = dyn_cast<FCmpInst>(NewFCmp)) {
     // Intersect FMF from the 2 source fcmps.
     NewFCmpInst->copyIRFlags(Op0);
     NewFCmpInst->andIRFlags(BO10);
   }
   return BinaryOperator::Create(Opcode, NewFCmp, BO11);
 }
 
 /// Match variations of De Morgan's Laws:
 /// (~A & ~B) == (~(A | B))
 /// (~A | ~B) == (~(A & B))
 static Instruction *matchDeMorgansLaws(BinaryOperator &I,
                                        InstCombiner::BuilderTy &Builder) {
   const Instruction::BinaryOps Opcode = I.getOpcode();
   assert((Opcode == Instruction::And || Opcode == Instruction::Or) &&
          "Trying to match De Morgan's Laws with something other than and/or");
 
   // Flip the logic operation.
   const Instruction::BinaryOps FlippedOpcode =
       (Opcode == Instruction::And) ? Instruction::Or : Instruction::And;
 
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
   Value *A, *B;
   if (match(Op0, m_OneUse(m_Not(m_Value(A)))) &&
       match(Op1, m_OneUse(m_Not(m_Value(B)))) &&
       !InstCombiner::isFreeToInvert(A, A->hasOneUse()) &&
       !InstCombiner::isFreeToInvert(B, B->hasOneUse())) {
     Value *AndOr =
         Builder.CreateBinOp(FlippedOpcode, A, B, I.getName() + ".demorgan");
     return BinaryOperator::CreateNot(AndOr);
   }
 
   // The 'not' ops may require reassociation.
   // (A & ~B) & ~C --> A & ~(B | C)
   // (~B & A) & ~C --> A & ~(B | C)
   // (A | ~B) | ~C --> A | ~(B & C)
   // (~B | A) | ~C --> A | ~(B & C)
   Value *C;
   if (match(Op0, m_OneUse(m_c_BinOp(Opcode, m_Value(A), m_Not(m_Value(B))))) &&
       match(Op1, m_Not(m_Value(C)))) {
     Value *FlippedBO = Builder.CreateBinOp(FlippedOpcode, B, C);
     return BinaryOperator::Create(Opcode, A, Builder.CreateNot(FlippedBO));
   }
 
   return nullptr;
 }
 
 bool InstCombinerImpl::shouldOptimizeCast(CastInst *CI) {
   Value *CastSrc = CI->getOperand(0);
 
   // Noop casts and casts of constants should be eliminated trivially.
   if (CI->getSrcTy() == CI->getDestTy() || isa<Constant>(CastSrc))
     return false;
 
   // If this cast is paired with another cast that can be eliminated, we prefer
   // to have it eliminated.
   if (const auto *PrecedingCI = dyn_cast<CastInst>(CastSrc))
     if (isEliminableCastPair(PrecedingCI, CI))
       return false;
 
   return true;
 }
 
 /// Fold {and,or,xor} (cast X), C.
 static Instruction *foldLogicCastConstant(BinaryOperator &Logic, CastInst *Cast,
                                           InstCombiner::BuilderTy &Builder) {
   Constant *C = dyn_cast<Constant>(Logic.getOperand(1));
   if (!C)
     return nullptr;
 
   auto LogicOpc = Logic.getOpcode();
   Type *DestTy = Logic.getType();
   Type *SrcTy = Cast->getSrcTy();
 
   // Move the logic operation ahead of a zext or sext if the constant is
   // unchanged in the smaller source type. Performing the logic in a smaller
   // type may provide more information to later folds, and the smaller logic
   // instruction may be cheaper (particularly in the case of vectors).
   Value *X;
   if (match(Cast, m_OneUse(m_ZExt(m_Value(X))))) {
     Constant *TruncC = ConstantExpr::getTrunc(C, SrcTy);
     Constant *ZextTruncC = ConstantExpr::getZExt(TruncC, DestTy);
     if (ZextTruncC == C) {
       // LogicOpc (zext X), C --> zext (LogicOpc X, C)
       Value *NewOp = Builder.CreateBinOp(LogicOpc, X, TruncC);
       return new ZExtInst(NewOp, DestTy);
     }
   }
 
   if (match(Cast, m_OneUse(m_SExt(m_Value(X))))) {
     Constant *TruncC = ConstantExpr::getTrunc(C, SrcTy);
     Constant *SextTruncC = ConstantExpr::getSExt(TruncC, DestTy);
     if (SextTruncC == C) {
       // LogicOpc (sext X), C --> sext (LogicOpc X, C)
       Value *NewOp = Builder.CreateBinOp(LogicOpc, X, TruncC);
       return new SExtInst(NewOp, DestTy);
     }
   }
 
   return nullptr;
 }
 
 /// Fold {and,or,xor} (cast X), Y.
 Instruction *InstCombinerImpl::foldCastedBitwiseLogic(BinaryOperator &I) {
   auto LogicOpc = I.getOpcode();
   assert(I.isBitwiseLogicOp() && "Unexpected opcode for bitwise logic folding");
 
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
   CastInst *Cast0 = dyn_cast<CastInst>(Op0);
   if (!Cast0)
     return nullptr;
 
   // This must be a cast from an integer or integer vector source type to allow
   // transformation of the logic operation to the source type.
   Type *DestTy = I.getType();
   Type *SrcTy = Cast0->getSrcTy();
   if (!SrcTy->isIntOrIntVectorTy())
     return nullptr;
 
   if (Instruction *Ret = foldLogicCastConstant(I, Cast0, Builder))
     return Ret;
 
   CastInst *Cast1 = dyn_cast<CastInst>(Op1);
   if (!Cast1)
     return nullptr;
 
   // Both operands of the logic operation are casts. The casts must be of the
   // same type for reduction.
   auto CastOpcode = Cast0->getOpcode();
   if (CastOpcode != Cast1->getOpcode() || SrcTy != Cast1->getSrcTy())
     return nullptr;
 
   Value *Cast0Src = Cast0->getOperand(0);
   Value *Cast1Src = Cast1->getOperand(0);
 
   // fold logic(cast(A), cast(B)) -> cast(logic(A, B))
   if (shouldOptimizeCast(Cast0) && shouldOptimizeCast(Cast1)) {
     Value *NewOp = Builder.CreateBinOp(LogicOpc, Cast0Src, Cast1Src,
                                         I.getName());
     return CastInst::Create(CastOpcode, NewOp, DestTy);
   }
 
   // For now, only 'and'/'or' have optimizations after this.
   if (LogicOpc == Instruction::Xor)
     return nullptr;
 
   // If this is logic(cast(icmp), cast(icmp)), try to fold this even if the
   // cast is otherwise not optimizable.  This happens for vector sexts.
   ICmpInst *ICmp0 = dyn_cast<ICmpInst>(Cast0Src);
   ICmpInst *ICmp1 = dyn_cast<ICmpInst>(Cast1Src);
   if (ICmp0 && ICmp1) {
     Value *Res = LogicOpc == Instruction::And ? foldAndOfICmps(ICmp0, ICmp1, I)
                                               : foldOrOfICmps(ICmp0, ICmp1, I);
     if (Res)
       return CastInst::Create(CastOpcode, Res, DestTy);
     return nullptr;
   }
 
   // If this is logic(cast(fcmp), cast(fcmp)), try to fold this even if the
   // cast is otherwise not optimizable.  This happens for vector sexts.
   FCmpInst *FCmp0 = dyn_cast<FCmpInst>(Cast0Src);
   FCmpInst *FCmp1 = dyn_cast<FCmpInst>(Cast1Src);
   if (FCmp0 && FCmp1)
     if (Value *R = foldLogicOfFCmps(FCmp0, FCmp1, LogicOpc == Instruction::And))
       return CastInst::Create(CastOpcode, R, DestTy);
 
   return nullptr;
 }
 
 static Instruction *foldAndToXor(BinaryOperator &I,
                                  InstCombiner::BuilderTy &Builder) {
   assert(I.getOpcode() == Instruction::And);
   Value *Op0 = I.getOperand(0);
   Value *Op1 = I.getOperand(1);
   Value *A, *B;
 
   // Operand complexity canonicalization guarantees that the 'or' is Op0.
   // (A | B) & ~(A & B) --> A ^ B
   // (A | B) & ~(B & A) --> A ^ B
   if (match(&I, m_BinOp(m_Or(m_Value(A), m_Value(B)),
                         m_Not(m_c_And(m_Deferred(A), m_Deferred(B))))))
     return BinaryOperator::CreateXor(A, B);
 
   // (A | ~B) & (~A | B) --> ~(A ^ B)
   // (A | ~B) & (B | ~A) --> ~(A ^ B)
   // (~B | A) & (~A | B) --> ~(A ^ B)
   // (~B | A) & (B | ~A) --> ~(A ^ B)
   if (Op0->hasOneUse() || Op1->hasOneUse())
     if (match(&I, m_BinOp(m_c_Or(m_Value(A), m_Not(m_Value(B))),
                           m_c_Or(m_Not(m_Deferred(A)), m_Deferred(B)))))
       return BinaryOperator::CreateNot(Builder.CreateXor(A, B));
 
   return nullptr;
 }
 
 static Instruction *foldOrToXor(BinaryOperator &I,
                                 InstCombiner::BuilderTy &Builder) {
   assert(I.getOpcode() == Instruction::Or);
   Value *Op0 = I.getOperand(0);
   Value *Op1 = I.getOperand(1);
   Value *A, *B;
 
   // Operand complexity canonicalization guarantees that the 'and' is Op0.
   // (A & B) | ~(A | B) --> ~(A ^ B)
   // (A & B) | ~(B | A) --> ~(A ^ B)
   if (Op0->hasOneUse() || Op1->hasOneUse())
     if (match(Op0, m_And(m_Value(A), m_Value(B))) &&
         match(Op1, m_Not(m_c_Or(m_Specific(A), m_Specific(B)))))
       return BinaryOperator::CreateNot(Builder.CreateXor(A, B));
 
   // Operand complexity canonicalization guarantees that the 'xor' is Op0.
   // (A ^ B) | ~(A | B) --> ~(A & B)
   // (A ^ B) | ~(B | A) --> ~(A & B)
   if (Op0->hasOneUse() || Op1->hasOneUse())
     if (match(Op0, m_Xor(m_Value(A), m_Value(B))) &&
         match(Op1, m_Not(m_c_Or(m_Specific(A), m_Specific(B)))))
       return BinaryOperator::CreateNot(Builder.CreateAnd(A, B));
 
   // (A & ~B) | (~A & B) --> A ^ B
   // (A & ~B) | (B & ~A) --> A ^ B
   // (~B & A) | (~A & B) --> A ^ B
   // (~B & A) | (B & ~A) --> A ^ B
   if (match(Op0, m_c_And(m_Value(A), m_Not(m_Value(B)))) &&
       match(Op1, m_c_And(m_Not(m_Specific(A)), m_Specific(B))))
     return BinaryOperator::CreateXor(A, B);
 
   return nullptr;
 }
 
 /// Return true if a constant shift amount is always less than the specified
 /// bit-width. If not, the shift could create poison in the narrower type.
 static bool canNarrowShiftAmt(Constant *C, unsigned BitWidth) {
   APInt Threshold(C->getType()->getScalarSizeInBits(), BitWidth);
   return match(C, m_SpecificInt_ICMP(ICmpInst::ICMP_ULT, Threshold));
 }
 
 /// Try to use narrower ops (sink zext ops) for an 'and' with binop operand and
 /// a common zext operand: and (binop (zext X), C), (zext X).
 Instruction *InstCombinerImpl::narrowMaskedBinOp(BinaryOperator &And) {
   // This transform could also apply to {or, and, xor}, but there are better
   // folds for those cases, so we don't expect those patterns here. AShr is not
   // handled because it should always be transformed to LShr in this sequence.
   // The subtract transform is different because it has a constant on the left.
   // Add/mul commute the constant to RHS; sub with constant RHS becomes add.
   Value *Op0 = And.getOperand(0), *Op1 = And.getOperand(1);
   Constant *C;
   if (!match(Op0, m_OneUse(m_Add(m_Specific(Op1), m_Constant(C)))) &&
       !match(Op0, m_OneUse(m_Mul(m_Specific(Op1), m_Constant(C)))) &&
       !match(Op0, m_OneUse(m_LShr(m_Specific(Op1), m_Constant(C)))) &&
       !match(Op0, m_OneUse(m_Shl(m_Specific(Op1), m_Constant(C)))) &&
       !match(Op0, m_OneUse(m_Sub(m_Constant(C), m_Specific(Op1)))))
     return nullptr;
 
   Value *X;
   if (!match(Op1, m_ZExt(m_Value(X))) || Op1->hasNUsesOrMore(3))
     return nullptr;
 
   Type *Ty = And.getType();
   if (!isa<VectorType>(Ty) && !shouldChangeType(Ty, X->getType()))
     return nullptr;
 
   // If we're narrowing a shift, the shift amount must be safe (less than the
   // width) in the narrower type. If the shift amount is greater, instsimplify
   // usually handles that case, but we can't guarantee/assert it.
   Instruction::BinaryOps Opc = cast<BinaryOperator>(Op0)->getOpcode();
   if (Opc == Instruction::LShr || Opc == Instruction::Shl)
     if (!canNarrowShiftAmt(C, X->getType()->getScalarSizeInBits()))
       return nullptr;
 
   // and (sub C, (zext X)), (zext X) --> zext (and (sub C', X), X)
   // and (binop (zext X), C), (zext X) --> zext (and (binop X, C'), X)
   Value *NewC = ConstantExpr::getTrunc(C, X->getType());
   Value *NewBO = Opc == Instruction::Sub ? Builder.CreateBinOp(Opc, NewC, X)
                                          : Builder.CreateBinOp(Opc, X, NewC);
   return new ZExtInst(Builder.CreateAnd(NewBO, X), Ty);
 }
 
 /// Try folding relatively complex patterns for both And and Or operations
 /// with all And and Or swapped.
 static Instruction *foldComplexAndOrPatterns(BinaryOperator &I,
                                              InstCombiner::BuilderTy &Builder) {
   const Instruction::BinaryOps Opcode = I.getOpcode();
   assert(Opcode == Instruction::And || Opcode == Instruction::Or);
 
   // Flip the logic operation.
   const Instruction::BinaryOps FlippedOpcode =
       (Opcode == Instruction::And) ? Instruction::Or : Instruction::And;
 
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
   Value *A, *B, *C, *X, *Y, *Dummy;
 
   // Match following expressions:
   // (~(A | B) & C)
   // (~(A & B) | C)
   // Captures X = ~(A | B) or ~(A & B)
   const auto matchNotOrAnd =
       [Opcode, FlippedOpcode](Value *Op, auto m_A, auto m_B, auto m_C,
                               Value *&X, bool CountUses = false) -> bool {
     if (CountUses && !Op->hasOneUse())
       return false;
 
     if (match(Op, m_c_BinOp(FlippedOpcode,
                             m_CombineAnd(m_Value(X),
                                          m_Not(m_c_BinOp(Opcode, m_A, m_B))),
                             m_C)))
       return !CountUses || X->hasOneUse();
 
     return false;
   };
 
   // (~(A | B) & C) | ... --> ...
   // (~(A & B) | C) & ... --> ...
   // TODO: One use checks are conservative. We just need to check that a total
   //       number of multiple used values does not exceed reduction
   //       in operations.
   if (matchNotOrAnd(Op0, m_Value(A), m_Value(B), m_Value(C), X)) {
     // (~(A | B) & C) | (~(A | C) & B) --> (B ^ C) & ~A
     // (~(A & B) | C) & (~(A & C) | B) --> ~((B ^ C) & A)
     if (matchNotOrAnd(Op1, m_Specific(A), m_Specific(C), m_Specific(B), Dummy,
                       true)) {
       Value *Xor = Builder.CreateXor(B, C);
       return (Opcode == Instruction::Or)
                  ? BinaryOperator::CreateAnd(Xor, Builder.CreateNot(A))
                  : BinaryOperator::CreateNot(Builder.CreateAnd(Xor, A));
     }
 
     // (~(A | B) & C) | (~(B | C) & A) --> (A ^ C) & ~B
     // (~(A & B) | C) & (~(B & C) | A) --> ~((A ^ C) & B)
     if (matchNotOrAnd(Op1, m_Specific(B), m_Specific(C), m_Specific(A), Dummy,
                       true)) {
       Value *Xor = Builder.CreateXor(A, C);
       return (Opcode == Instruction::Or)
                  ? BinaryOperator::CreateAnd(Xor, Builder.CreateNot(B))
                  : BinaryOperator::CreateNot(Builder.CreateAnd(Xor, B));
     }
 
     // (~(A | B) & C) | ~(A | C) --> ~((B & C) | A)
     // (~(A & B) | C) & ~(A & C) --> ~((B | C) & A)
     if (match(Op1, m_OneUse(m_Not(m_OneUse(
                        m_c_BinOp(Opcode, m_Specific(A), m_Specific(C)))))))
       return BinaryOperator::CreateNot(Builder.CreateBinOp(
           Opcode, Builder.CreateBinOp(FlippedOpcode, B, C), A));
 
     // (~(A | B) & C) | ~(B | C) --> ~((A & C) | B)
     // (~(A & B) | C) & ~(B & C) --> ~((A | C) & B)
     if (match(Op1, m_OneUse(m_Not(m_OneUse(
                        m_c_BinOp(Opcode, m_Specific(B), m_Specific(C)))))))
       return BinaryOperator::CreateNot(Builder.CreateBinOp(
           Opcode, Builder.CreateBinOp(FlippedOpcode, A, C), B));
 
     // (~(A | B) & C) | ~(C | (A ^ B)) --> ~((A | B) & (C | (A ^ B)))
     // Note, the pattern with swapped and/or is not handled because the
     // result is more undefined than a source:
     // (~(A & B) | C) & ~(C & (A ^ B)) --> (A ^ B ^ C) | ~(A | C) is invalid.
     if (Opcode == Instruction::Or && Op0->hasOneUse() &&
         match(Op1, m_OneUse(m_Not(m_CombineAnd(
                        m_Value(Y),
                        m_c_BinOp(Opcode, m_Specific(C),
                                  m_c_Xor(m_Specific(A), m_Specific(B)))))))) {
       // X = ~(A | B)
       // Y = (C | (A ^ B)
       Value *Or = cast<BinaryOperator>(X)->getOperand(0);
       return BinaryOperator::CreateNot(Builder.CreateAnd(Or, Y));
     }
   }
 
   // (~A & B & C) | ... --> ...
   // (~A | B | C) | ... --> ...
   // TODO: One use checks are conservative. We just need to check that a total
   //       number of multiple used values does not exceed reduction
   //       in operations.
   if (match(Op0,
             m_OneUse(m_c_BinOp(FlippedOpcode,
                                m_BinOp(FlippedOpcode, m_Value(B), m_Value(C)),
                                m_CombineAnd(m_Value(X), m_Not(m_Value(A)))))) ||
       match(Op0, m_OneUse(m_c_BinOp(
                      FlippedOpcode,
                      m_c_BinOp(FlippedOpcode, m_Value(C),
                                m_CombineAnd(m_Value(X), m_Not(m_Value(A)))),
                      m_Value(B))))) {
     // X = ~A
     // (~A & B & C) | ~(A | B | C) --> ~(A | (B ^ C))
     // (~A | B | C) & ~(A & B & C) --> (~A | (B ^ C))
     if (match(Op1, m_OneUse(m_Not(m_c_BinOp(
                        Opcode, m_c_BinOp(Opcode, m_Specific(A), m_Specific(B)),
                        m_Specific(C))))) ||
         match(Op1, m_OneUse(m_Not(m_c_BinOp(
                        Opcode, m_c_BinOp(Opcode, m_Specific(B), m_Specific(C)),
                        m_Specific(A))))) ||
         match(Op1, m_OneUse(m_Not(m_c_BinOp(
                        Opcode, m_c_BinOp(Opcode, m_Specific(A), m_Specific(C)),
                        m_Specific(B)))))) {
       Value *Xor = Builder.CreateXor(B, C);
       return (Opcode == Instruction::Or)
                  ? BinaryOperator::CreateNot(Builder.CreateOr(Xor, A))
                  : BinaryOperator::CreateOr(Xor, X);
     }
 
     // (~A & B & C) | ~(A | B) --> (C | ~B) & ~A
     // (~A | B | C) & ~(A & B) --> (C & ~B) | ~A
     if (match(Op1, m_OneUse(m_Not(m_OneUse(
                        m_c_BinOp(Opcode, m_Specific(A), m_Specific(B)))))))
       return BinaryOperator::Create(
           FlippedOpcode, Builder.CreateBinOp(Opcode, C, Builder.CreateNot(B)),
           X);
 
     // (~A & B & C) | ~(A | C) --> (B | ~C) & ~A
     // (~A | B | C) & ~(A & C) --> (B & ~C) | ~A
     if (match(Op1, m_OneUse(m_Not(m_OneUse(
                        m_c_BinOp(Opcode, m_Specific(A), m_Specific(C)))))))
       return BinaryOperator::Create(
           FlippedOpcode, Builder.CreateBinOp(Opcode, B, Builder.CreateNot(C)),
           X);
   }
 
   return nullptr;
 }
 
 // FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
 // here. We should standardize that construct where it is needed or choose some
 // other way to ensure that commutated variants of patterns are not missed.
 Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
   Type *Ty = I.getType();
 
   if (Value *V = SimplifyAndInst(I.getOperand(0), I.getOperand(1),
                                  SQ.getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
 
   if (SimplifyAssociativeOrCommutative(I))
     return &I;
 
   if (Instruction *X = foldVectorBinop(I))
     return X;
 
   if (Instruction *Phi = foldBinopWithPhiOperands(I))
     return Phi;
 
   // See if we can simplify any instructions used by the instruction whose sole
   // purpose is to compute bits we don't care about.
   if (SimplifyDemandedInstructionBits(I))
     return &I;
 
   // Do this before using distributive laws to catch simple and/or/not patterns.
   if (Instruction *Xor = foldAndToXor(I, Builder))
     return Xor;
 
   if (Instruction *X = foldComplexAndOrPatterns(I, Builder))
     return X;
 
   // (A|B)&(A|C) -> A|(B&C) etc
   if (Value *V = SimplifyUsingDistributiveLaws(I))
     return replaceInstUsesWith(I, V);
 
   if (Value *V = SimplifyBSwap(I, Builder))
     return replaceInstUsesWith(I, V);
 
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
 
   Value *X, *Y;
   if (match(Op0, m_OneUse(m_LogicalShift(m_One(), m_Value(X)))) &&
       match(Op1, m_One())) {
     // (1 << X) & 1 --> zext(X == 0)
     // (1 >> X) & 1 --> zext(X == 0)
     Value *IsZero = Builder.CreateICmpEQ(X, ConstantInt::get(Ty, 0));
     return new ZExtInst(IsZero, Ty);
   }
 
   const APInt *C;
   if (match(Op1, m_APInt(C))) {
     const APInt *XorC;
     if (match(Op0, m_OneUse(m_Xor(m_Value(X), m_APInt(XorC))))) {
       // (X ^ C1) & C2 --> (X & C2) ^ (C1&C2)
       Constant *NewC = ConstantInt::get(Ty, *C & *XorC);
       Value *And = Builder.CreateAnd(X, Op1);
       And->takeName(Op0);
       return BinaryOperator::CreateXor(And, NewC);
     }
 
     const APInt *OrC;
     if (match(Op0, m_OneUse(m_Or(m_Value(X), m_APInt(OrC))))) {
       // (X | C1) & C2 --> (X & C2^(C1&C2)) | (C1&C2)
       // NOTE: This reduces the number of bits set in the & mask, which
       // can expose opportunities for store narrowing for scalars.
       // NOTE: SimplifyDemandedBits should have already removed bits from C1
       // that aren't set in C2. Meaning we can replace (C1&C2) with C1 in
       // above, but this feels safer.
       APInt Together = *C & *OrC;
       Value *And = Builder.CreateAnd(X, ConstantInt::get(Ty, Together ^ *C));
       And->takeName(Op0);
       return BinaryOperator::CreateOr(And, ConstantInt::get(Ty, Together));
     }
 
     // If the mask is only needed on one incoming arm, push the 'and' op up.
     if (match(Op0, m_OneUse(m_Xor(m_Value(X), m_Value(Y)))) ||
         match(Op0, m_OneUse(m_Or(m_Value(X), m_Value(Y))))) {
       APInt NotAndMask(~(*C));
       BinaryOperator::BinaryOps BinOp = cast<BinaryOperator>(Op0)->getOpcode();
       if (MaskedValueIsZero(X, NotAndMask, 0, &I)) {
         // Not masking anything out for the LHS, move mask to RHS.
         // and ({x}or X, Y), C --> {x}or X, (and Y, C)
         Value *NewRHS = Builder.CreateAnd(Y, Op1, Y->getName() + ".masked");
         return BinaryOperator::Create(BinOp, X, NewRHS);
       }
       if (!isa<Constant>(Y) && MaskedValueIsZero(Y, NotAndMask, 0, &I)) {
         // Not masking anything out for the RHS, move mask to LHS.
         // and ({x}or X, Y), C --> {x}or (and X, C), Y
         Value *NewLHS = Builder.CreateAnd(X, Op1, X->getName() + ".masked");
         return BinaryOperator::Create(BinOp, NewLHS, Y);
       }
     }
 
     unsigned Width = Ty->getScalarSizeInBits();
     const APInt *ShiftC;
     if (match(Op0, m_OneUse(m_SExt(m_AShr(m_Value(X), m_APInt(ShiftC)))))) {
       if (*C == APInt::getLowBitsSet(Width, Width - ShiftC->getZExtValue())) {
         // We are clearing high bits that were potentially set by sext+ashr:
         // and (sext (ashr X, ShiftC)), C --> lshr (sext X), ShiftC
         Value *Sext = Builder.CreateSExt(X, Ty);
         Constant *ShAmtC = ConstantInt::get(Ty, ShiftC->zext(Width));
         return BinaryOperator::CreateLShr(Sext, ShAmtC);
       }
     }
 
     // If this 'and' clears the sign-bits added by ashr, replace with lshr:
     // and (ashr X, ShiftC), C --> lshr X, ShiftC
     if (match(Op0, m_AShr(m_Value(X), m_APInt(ShiftC))) && ShiftC->ult(Width) &&
         C->isMask(Width - ShiftC->getZExtValue()))
       return BinaryOperator::CreateLShr(X, ConstantInt::get(Ty, *ShiftC));
 
     const APInt *AddC;
     if (match(Op0, m_Add(m_Value(X), m_APInt(AddC)))) {
       // If we add zeros to every bit below a mask, the add has no effect:
       // (X + AddC) & LowMaskC --> X & LowMaskC
       unsigned Ctlz = C->countLeadingZeros();
       APInt LowMask(APInt::getLowBitsSet(Width, Width - Ctlz));
       if ((*AddC & LowMask).isZero())
         return BinaryOperator::CreateAnd(X, Op1);
 
       // If we are masking the result of the add down to exactly one bit and
       // the constant we are adding has no bits set below that bit, then the
       // add is flipping a single bit. Example:
       // (X + 4) & 4 --> (X & 4) ^ 4
       if (Op0->hasOneUse() && C->isPowerOf2() && (*AddC & (*C - 1)) == 0) {
         assert((*C & *AddC) != 0 && "Expected common bit");
         Value *NewAnd = Builder.CreateAnd(X, Op1);
         return BinaryOperator::CreateXor(NewAnd, Op1);
       }
     }
 
     // ((C1 OP zext(X)) & C2) -> zext((C1 OP X) & C2) if C2 fits in the
     // bitwidth of X and OP behaves well when given trunc(C1) and X.
     auto isSuitableBinOpcode = [](BinaryOperator *B) {
       switch (B->getOpcode()) {
       case Instruction::Xor:
       case Instruction::Or:
       case Instruction::Mul:
       case Instruction::Add:
       case Instruction::Sub:
         return true;
       default:
         return false;
       }
     };
     BinaryOperator *BO;
     if (match(Op0, m_OneUse(m_BinOp(BO))) && isSuitableBinOpcode(BO)) {
       Value *X;
       const APInt *C1;
       // TODO: The one-use restrictions could be relaxed a little if the AND
       // is going to be removed.
       if (match(BO, m_c_BinOp(m_OneUse(m_ZExt(m_Value(X))), m_APInt(C1))) &&
           C->isIntN(X->getType()->getScalarSizeInBits())) {
         unsigned XWidth = X->getType()->getScalarSizeInBits();
         Constant *TruncC1 = ConstantInt::get(X->getType(), C1->trunc(XWidth));
         Value *BinOp = isa<ZExtInst>(BO->getOperand(0))
                            ? Builder.CreateBinOp(BO->getOpcode(), X, TruncC1)
                            : Builder.CreateBinOp(BO->getOpcode(), TruncC1, X);
         Constant *TruncC = ConstantInt::get(X->getType(), C->trunc(XWidth));
         Value *And = Builder.CreateAnd(BinOp, TruncC);
         return new ZExtInst(And, Ty);
       }
     }
   }
 
   if (match(&I, m_And(m_OneUse(m_Shl(m_ZExt(m_Value(X)), m_Value(Y))),
                       m_SignMask())) &&
       match(Y, m_SpecificInt_ICMP(
                    ICmpInst::Predicate::ICMP_EQ,
                    APInt(Ty->getScalarSizeInBits(),
                          Ty->getScalarSizeInBits() -
                              X->getType()->getScalarSizeInBits())))) {
     auto *SExt = Builder.CreateSExt(X, Ty, X->getName() + ".signext");
     auto *SanitizedSignMask = cast<Constant>(Op1);
     // We must be careful with the undef elements of the sign bit mask, however:
     // the mask elt can be undef iff the shift amount for that lane was undef,
     // otherwise we need to sanitize undef masks to zero.
     SanitizedSignMask = Constant::replaceUndefsWith(
         SanitizedSignMask, ConstantInt::getNullValue(Ty->getScalarType()));
     SanitizedSignMask =
         Constant::mergeUndefsWith(SanitizedSignMask, cast<Constant>(Y));
     return BinaryOperator::CreateAnd(SExt, SanitizedSignMask);
   }
 
   if (Instruction *Z = narrowMaskedBinOp(I))
     return Z;
 
   if (I.getType()->isIntOrIntVectorTy(1)) {
     if (auto *SI0 = dyn_cast<SelectInst>(Op0)) {
       if (auto *I =
               foldAndOrOfSelectUsingImpliedCond(Op1, *SI0, /* IsAnd */ true))
         return I;
     }
     if (auto *SI1 = dyn_cast<SelectInst>(Op1)) {
       if (auto *I =
               foldAndOrOfSelectUsingImpliedCond(Op0, *SI1, /* IsAnd */ true))
         return I;
     }
   }
 
   if (Instruction *FoldedLogic = foldBinOpIntoSelectOrPhi(I))
     return FoldedLogic;
 
   if (Instruction *DeMorgan = matchDeMorgansLaws(I, Builder))
     return DeMorgan;
 
   {
     Value *A, *B, *C;
     // A & (A ^ B) --> A & ~B
     if (match(Op1, m_OneUse(m_c_Xor(m_Specific(Op0), m_Value(B)))))
       return BinaryOperator::CreateAnd(Op0, Builder.CreateNot(B));
     // (A ^ B) & A --> A & ~B
     if (match(Op0, m_OneUse(m_c_Xor(m_Specific(Op1), m_Value(B)))))
       return BinaryOperator::CreateAnd(Op1, Builder.CreateNot(B));
 
     // A & ~(A ^ B) --> A & B
     if (match(Op1, m_Not(m_c_Xor(m_Specific(Op0), m_Value(B)))))
       return BinaryOperator::CreateAnd(Op0, B);
     // ~(A ^ B) & A --> A & B
     if (match(Op0, m_Not(m_c_Xor(m_Specific(Op1), m_Value(B)))))
       return BinaryOperator::CreateAnd(Op1, B);
 
     // (A ^ B) & ((B ^ C) ^ A) -> (A ^ B) & ~C
     if (match(Op0, m_Xor(m_Value(A), m_Value(B))))
       if (match(Op1, m_Xor(m_Xor(m_Specific(B), m_Value(C)), m_Specific(A))))
         if (Op1->hasOneUse() || isFreeToInvert(C, C->hasOneUse()))
           return BinaryOperator::CreateAnd(Op0, Builder.CreateNot(C));
 
     // ((A ^ C) ^ B) & (B ^ A) -> (B ^ A) & ~C
     if (match(Op0, m_Xor(m_Xor(m_Value(A), m_Value(C)), m_Value(B))))
       if (match(Op1, m_Xor(m_Specific(B), m_Specific(A))))
         if (Op0->hasOneUse() || isFreeToInvert(C, C->hasOneUse()))
           return BinaryOperator::CreateAnd(Op1, Builder.CreateNot(C));
 
     // (A | B) & (~A ^ B) -> A & B
     // (A | B) & (B ^ ~A) -> A & B
     // (B | A) & (~A ^ B) -> A & B
     // (B | A) & (B ^ ~A) -> A & B
     if (match(Op1, m_c_Xor(m_Not(m_Value(A)), m_Value(B))) &&
         match(Op0, m_c_Or(m_Specific(A), m_Specific(B))))
       return BinaryOperator::CreateAnd(A, B);
 
     // (~A ^ B) & (A | B) -> A & B
     // (~A ^ B) & (B | A) -> A & B
     // (B ^ ~A) & (A | B) -> A & B
     // (B ^ ~A) & (B | A) -> A & B
     if (match(Op0, m_c_Xor(m_Not(m_Value(A)), m_Value(B))) &&
         match(Op1, m_c_Or(m_Specific(A), m_Specific(B))))
       return BinaryOperator::CreateAnd(A, B);
 
     // (~A | B) & (A ^ B) -> ~A & B
     // (~A | B) & (B ^ A) -> ~A & B
     // (B | ~A) & (A ^ B) -> ~A & B
     // (B | ~A) & (B ^ A) -> ~A & B
     if (match(Op0, m_c_Or(m_Not(m_Value(A)), m_Value(B))) &&
         match(Op1, m_c_Xor(m_Specific(A), m_Specific(B))))
       return BinaryOperator::CreateAnd(Builder.CreateNot(A), B);
 
     // (A ^ B) & (~A | B) -> ~A & B
     // (B ^ A) & (~A | B) -> ~A & B
     // (A ^ B) & (B | ~A) -> ~A & B
     // (B ^ A) & (B | ~A) -> ~A & B
     if (match(Op1, m_c_Or(m_Not(m_Value(A)), m_Value(B))) &&
         match(Op0, m_c_Xor(m_Specific(A), m_Specific(B))))
       return BinaryOperator::CreateAnd(Builder.CreateNot(A), B);
   }
 
   {
     ICmpInst *LHS = dyn_cast<ICmpInst>(Op0);
     ICmpInst *RHS = dyn_cast<ICmpInst>(Op1);
     if (LHS && RHS)
       if (Value *Res = foldAndOfICmps(LHS, RHS, I))
         return replaceInstUsesWith(I, Res);
 
     // TODO: Make this recursive; it's a little tricky because an arbitrary
     // number of 'and' instructions might have to be created.
     if (LHS && match(Op1, m_OneUse(m_And(m_Value(X), m_Value(Y))))) {
       if (auto *Cmp = dyn_cast<ICmpInst>(X))
         if (Value *Res = foldAndOfICmps(LHS, Cmp, I))
           return replaceInstUsesWith(I, Builder.CreateAnd(Res, Y));
       if (auto *Cmp = dyn_cast<ICmpInst>(Y))
         if (Value *Res = foldAndOfICmps(LHS, Cmp, I))
           return replaceInstUsesWith(I, Builder.CreateAnd(Res, X));
     }
     if (RHS && match(Op0, m_OneUse(m_And(m_Value(X), m_Value(Y))))) {
       if (auto *Cmp = dyn_cast<ICmpInst>(X))
         if (Value *Res = foldAndOfICmps(Cmp, RHS, I))
           return replaceInstUsesWith(I, Builder.CreateAnd(Res, Y));
       if (auto *Cmp = dyn_cast<ICmpInst>(Y))
         if (Value *Res = foldAndOfICmps(Cmp, RHS, I))
           return replaceInstUsesWith(I, Builder.CreateAnd(Res, X));
     }
   }
 
   if (FCmpInst *LHS = dyn_cast<FCmpInst>(I.getOperand(0)))
     if (FCmpInst *RHS = dyn_cast<FCmpInst>(I.getOperand(1)))
       if (Value *Res = foldLogicOfFCmps(LHS, RHS, true))
         return replaceInstUsesWith(I, Res);
 
   if (Instruction *FoldedFCmps = reassociateFCmps(I, Builder))
     return FoldedFCmps;
 
   if (Instruction *CastedAnd = foldCastedBitwiseLogic(I))
     return CastedAnd;
 
   if (Instruction *Sel = foldBinopOfSextBoolToSelect(I))
     return Sel;
 
   // and(sext(A), B) / and(B, sext(A)) --> A ? B : 0, where A is i1 or <N x i1>.
   // TODO: Move this into foldBinopOfSextBoolToSelect as a more generalized fold
   //       with binop identity constant. But creating a select with non-constant
   //       arm may not be reversible due to poison semantics. Is that a good
   //       canonicalization?
   Value *A;
   if (match(Op0, m_OneUse(m_SExt(m_Value(A)))) &&
       A->getType()->isIntOrIntVectorTy(1))
     return SelectInst::Create(A, Op1, Constant::getNullValue(Ty));
   if (match(Op1, m_OneUse(m_SExt(m_Value(A)))) &&
       A->getType()->isIntOrIntVectorTy(1))
     return SelectInst::Create(A, Op0, Constant::getNullValue(Ty));
 
   // (iN X s>> (N-1)) & Y --> (X s< 0) ? Y : 0
   unsigned FullShift = Ty->getScalarSizeInBits() - 1;
   if (match(&I, m_c_And(m_OneUse(m_AShr(m_Value(X), m_SpecificInt(FullShift))),
                         m_Value(Y)))) {
     Constant *Zero = ConstantInt::getNullValue(Ty);
     Value *Cmp = Builder.CreateICmpSLT(X, Zero, "isneg");
     return SelectInst::Create(Cmp, Y, Zero);
   }
   // If there's a 'not' of the shifted value, swap the select operands:
   // ~(iN X s>> (N-1)) & Y --> (X s< 0) ? 0 : Y
   if (match(&I, m_c_And(m_OneUse(m_Not(
                             m_AShr(m_Value(X), m_SpecificInt(FullShift)))),
                         m_Value(Y)))) {
     Constant *Zero = ConstantInt::getNullValue(Ty);
     Value *Cmp = Builder.CreateICmpSLT(X, Zero, "isneg");
     return SelectInst::Create(Cmp, Zero, Y);
   }
 
   // (~x) & y  -->  ~(x | (~y))  iff that gets rid of inversions
   if (sinkNotIntoOtherHandOfAndOrOr(I))
     return &I;
 
   // An and recurrence w/loop invariant step is equivelent to (and start, step)
   PHINode *PN = nullptr;
   Value *Start = nullptr, *Step = nullptr;
   if (matchSimpleRecurrence(&I, PN, Start, Step) && DT.dominates(Step, PN))
     return replaceInstUsesWith(I, Builder.CreateAnd(Start, Step));
 
   return nullptr;
 }
 
 Instruction *InstCombinerImpl::matchBSwapOrBitReverse(Instruction &I,
                                                       bool MatchBSwaps,
                                                       bool MatchBitReversals) {
   SmallVector<Instruction *, 4> Insts;
   if (!recognizeBSwapOrBitReverseIdiom(&I, MatchBSwaps, MatchBitReversals,
                                        Insts))
     return nullptr;
   Instruction *LastInst = Insts.pop_back_val();
   LastInst->removeFromParent();
 
   for (auto *Inst : Insts)
     Worklist.push(Inst);
   return LastInst;
 }
 
 /// Match UB-safe variants of the funnel shift intrinsic.
 static Instruction *matchFunnelShift(Instruction &Or, InstCombinerImpl &IC) {
   // TODO: Can we reduce the code duplication between this and the related
   // rotate matching code under visitSelect and visitTrunc?
   unsigned Width = Or.getType()->getScalarSizeInBits();
 
   // First, find an or'd pair of opposite shifts:
   // or (lshr ShVal0, ShAmt0), (shl ShVal1, ShAmt1)
   BinaryOperator *Or0, *Or1;
   if (!match(Or.getOperand(0), m_BinOp(Or0)) ||
       !match(Or.getOperand(1), m_BinOp(Or1)))
     return nullptr;
 
   Value *ShVal0, *ShVal1, *ShAmt0, *ShAmt1;
   if (!match(Or0, m_OneUse(m_LogicalShift(m_Value(ShVal0), m_Value(ShAmt0)))) ||
       !match(Or1, m_OneUse(m_LogicalShift(m_Value(ShVal1), m_Value(ShAmt1)))) ||
       Or0->getOpcode() == Or1->getOpcode())
     return nullptr;
 
   // Canonicalize to or(shl(ShVal0, ShAmt0), lshr(ShVal1, ShAmt1)).
   if (Or0->getOpcode() == BinaryOperator::LShr) {
     std::swap(Or0, Or1);
     std::swap(ShVal0, ShVal1);
     std::swap(ShAmt0, ShAmt1);
   }
   assert(Or0->getOpcode() == BinaryOperator::Shl &&
          Or1->getOpcode() == BinaryOperator::LShr &&
          "Illegal or(shift,shift) pair");
 
   // Match the shift amount operands for a funnel shift pattern. This always
   // matches a subtraction on the R operand.
   auto matchShiftAmount = [&](Value *L, Value *R, unsigned Width) -> Value * {
     // Check for constant shift amounts that sum to the bitwidth.
     const APInt *LI, *RI;
     if (match(L, m_APIntAllowUndef(LI)) && match(R, m_APIntAllowUndef(RI)))
       if (LI->ult(Width) && RI->ult(Width) && (*LI + *RI) == Width)
         return ConstantInt::get(L->getType(), *LI);
 
     Constant *LC, *RC;
     if (match(L, m_Constant(LC)) && match(R, m_Constant(RC)) &&
         match(L, m_SpecificInt_ICMP(ICmpInst::ICMP_ULT, APInt(Width, Width))) &&
         match(R, m_SpecificInt_ICMP(ICmpInst::ICMP_ULT, APInt(Width, Width))) &&
         match(ConstantExpr::getAdd(LC, RC), m_SpecificIntAllowUndef(Width)))
       return ConstantExpr::mergeUndefsWith(LC, RC);
 
     // (shl ShVal, X) | (lshr ShVal, (Width - x)) iff X < Width.
     // We limit this to X < Width in case the backend re-expands the intrinsic,
     // and has to reintroduce a shift modulo operation (InstCombine might remove
     // it after this fold). This still doesn't guarantee that the final codegen
     // will match this original pattern.
     if (match(R, m_OneUse(m_Sub(m_SpecificInt(Width), m_Specific(L))))) {
       KnownBits KnownL = IC.computeKnownBits(L, /*Depth*/ 0, &Or);
       return KnownL.getMaxValue().ult(Width) ? L : nullptr;
     }
 
     // For non-constant cases, the following patterns currently only work for
     // rotation patterns.
     // TODO: Add general funnel-shift compatible patterns.
     if (ShVal0 != ShVal1)
       return nullptr;
 
     // For non-constant cases we don't support non-pow2 shift masks.
     // TODO: Is it worth matching urem as well?
     if (!isPowerOf2_32(Width))
       return nullptr;
 
     // The shift amount may be masked with negation:
     // (shl ShVal, (X & (Width - 1))) | (lshr ShVal, ((-X) & (Width - 1)))
     Value *X;
     unsigned Mask = Width - 1;
     if (match(L, m_And(m_Value(X), m_SpecificInt(Mask))) &&
         match(R, m_And(m_Neg(m_Specific(X)), m_SpecificInt(Mask))))
       return X;
 
     // Similar to above, but the shift amount may be extended after masking,
     // so return the extended value as the parameter for the intrinsic.
     if (match(L, m_ZExt(m_And(m_Value(X), m_SpecificInt(Mask)))) &&
         match(R, m_And(m_Neg(m_ZExt(m_And(m_Specific(X), m_SpecificInt(Mask)))),
                        m_SpecificInt(Mask))))
       return L;
 
     if (match(L, m_ZExt(m_And(m_Value(X), m_SpecificInt(Mask)))) &&
         match(R, m_ZExt(m_And(m_Neg(m_Specific(X)), m_SpecificInt(Mask)))))
       return L;
 
     return nullptr;
   };
 
   Value *ShAmt = matchShiftAmount(ShAmt0, ShAmt1, Width);
   bool IsFshl = true; // Sub on LSHR.
   if (!ShAmt) {
     ShAmt = matchShiftAmount(ShAmt1, ShAmt0, Width);
     IsFshl = false; // Sub on SHL.
   }
   if (!ShAmt)
     return nullptr;
 
   Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
   Function *F = Intrinsic::getDeclaration(Or.getModule(), IID, Or.getType());
   return CallInst::Create(F, {ShVal0, ShVal1, ShAmt});
 }
 
 /// Attempt to combine or(zext(x),shl(zext(y),bw/2) concat packing patterns.
 static Instruction *matchOrConcat(Instruction &Or,
                                   InstCombiner::BuilderTy &Builder) {
   assert(Or.getOpcode() == Instruction::Or && "bswap requires an 'or'");
   Value *Op0 = Or.getOperand(0), *Op1 = Or.getOperand(1);
   Type *Ty = Or.getType();
 
   unsigned Width = Ty->getScalarSizeInBits();
   if ((Width & 1) != 0)
     return nullptr;
   unsigned HalfWidth = Width / 2;
 
   // Canonicalize zext (lower half) to LHS.
   if (!isa<ZExtInst>(Op0))
     std::swap(Op0, Op1);
 
   // Find lower/upper half.
   Value *LowerSrc, *ShlVal, *UpperSrc;
   const APInt *C;
   if (!match(Op0, m_OneUse(m_ZExt(m_Value(LowerSrc)))) ||
       !match(Op1, m_OneUse(m_Shl(m_Value(ShlVal), m_APInt(C)))) ||
       !match(ShlVal, m_OneUse(m_ZExt(m_Value(UpperSrc)))))
     return nullptr;
   if (*C != HalfWidth || LowerSrc->getType() != UpperSrc->getType() ||
       LowerSrc->getType()->getScalarSizeInBits() != HalfWidth)
     return nullptr;
 
   auto ConcatIntrinsicCalls = [&](Intrinsic::ID id, Value *Lo, Value *Hi) {
     Value *NewLower = Builder.CreateZExt(Lo, Ty);
     Value *NewUpper = Builder.CreateZExt(Hi, Ty);
     NewUpper = Builder.CreateShl(NewUpper, HalfWidth);
     Value *BinOp = Builder.CreateOr(NewLower, NewUpper);
     Function *F = Intrinsic::getDeclaration(Or.getModule(), id, Ty);
     return Builder.CreateCall(F, BinOp);
   };
 
   // BSWAP: Push the concat down, swapping the lower/upper sources.
   // concat(bswap(x),bswap(y)) -> bswap(concat(x,y))
   Value *LowerBSwap, *UpperBSwap;
   if (match(LowerSrc, m_BSwap(m_Value(LowerBSwap))) &&
       match(UpperSrc, m_BSwap(m_Value(UpperBSwap))))
     return ConcatIntrinsicCalls(Intrinsic::bswap, UpperBSwap, LowerBSwap);
 
   // BITREVERSE: Push the concat down, swapping the lower/upper sources.
   // concat(bitreverse(x),bitreverse(y)) -> bitreverse(concat(x,y))
   Value *LowerBRev, *UpperBRev;
   if (match(LowerSrc, m_BitReverse(m_Value(LowerBRev))) &&
       match(UpperSrc, m_BitReverse(m_Value(UpperBRev))))
     return ConcatIntrinsicCalls(Intrinsic::bitreverse, UpperBRev, LowerBRev);
 
   return nullptr;
 }
 
 /// If all elements of two constant vectors are 0/-1 and inverses, return true.
 static bool areInverseVectorBitmasks(Constant *C1, Constant *C2) {
   unsigned NumElts = cast<FixedVectorType>(C1->getType())->getNumElements();
   for (unsigned i = 0; i != NumElts; ++i) {
     Constant *EltC1 = C1->getAggregateElement(i);
     Constant *EltC2 = C2->getAggregateElement(i);
     if (!EltC1 || !EltC2)
       return false;
 
     // One element must be all ones, and the other must be all zeros.
     if (!((match(EltC1, m_Zero()) && match(EltC2, m_AllOnes())) ||
           (match(EltC2, m_Zero()) && match(EltC1, m_AllOnes()))))
       return false;
   }
   return true;
 }
 
 /// We have an expression of the form (A & C) | (B & D). If A is a scalar or
 /// vector composed of all-zeros or all-ones values and is the bitwise 'not' of
 /// B, it can be used as the condition operand of a select instruction.
 Value *InstCombinerImpl::getSelectCondition(Value *A, Value *B) {
   // We may have peeked through bitcasts in the caller.
   // Exit immediately if we don't have (vector) integer types.
   Type *Ty = A->getType();
   if (!Ty->isIntOrIntVectorTy() || !B->getType()->isIntOrIntVectorTy())
     return nullptr;
 
   // If A is the 'not' operand of B and has enough signbits, we have our answer.
   if (match(B, m_Not(m_Specific(A)))) {
     // If these are scalars or vectors of i1, A can be used directly.
     if (Ty->isIntOrIntVectorTy(1))
       return A;
 
     // If we look through a vector bitcast, the caller will bitcast the operands
     // to match the condition's number of bits (N x i1).
     // To make this poison-safe, disallow bitcast from wide element to narrow
     // element. That could allow poison in lanes where it was not present in the
     // original code.
     A = peekThroughBitcast(A);
     if (A->getType()->isIntOrIntVectorTy()) {
       unsigned NumSignBits = ComputeNumSignBits(A);
       if (NumSignBits == A->getType()->getScalarSizeInBits() &&
           NumSignBits <= Ty->getScalarSizeInBits())
         return Builder.CreateTrunc(A, CmpInst::makeCmpResultType(A->getType()));
     }
     return nullptr;
   }
 
   // If both operands are constants, see if the constants are inverse bitmasks.
   Constant *AConst, *BConst;
   if (match(A, m_Constant(AConst)) && match(B, m_Constant(BConst)))
     if (AConst == ConstantExpr::getNot(BConst) &&
         ComputeNumSignBits(A) == Ty->getScalarSizeInBits())
       return Builder.CreateZExtOrTrunc(A, CmpInst::makeCmpResultType(Ty));
 
   // Look for more complex patterns. The 'not' op may be hidden behind various
   // casts. Look through sexts and bitcasts to find the booleans.
   Value *Cond;
   Value *NotB;
   if (match(A, m_SExt(m_Value(Cond))) &&
       Cond->getType()->isIntOrIntVectorTy(1)) {
     // A = sext i1 Cond; B = sext (not (i1 Cond))
     if (match(B, m_SExt(m_Not(m_Specific(Cond)))))
       return Cond;
 
     // A = sext i1 Cond; B = not ({bitcast} (sext (i1 Cond)))
     // TODO: The one-use checks are unnecessary or misplaced. If the caller
     //       checked for uses on logic ops/casts, that should be enough to
     //       make this transform worthwhile.
     if (match(B, m_OneUse(m_Not(m_Value(NotB))))) {
       NotB = peekThroughBitcast(NotB, true);
       if (match(NotB, m_SExt(m_Specific(Cond))))
         return Cond;
     }
   }
 
   // All scalar (and most vector) possibilities should be handled now.
   // Try more matches that only apply to non-splat constant vectors.
   if (!Ty->isVectorTy())
     return nullptr;
 
   // If both operands are xor'd with constants using the same sexted boolean
   // operand, see if the constants are inverse bitmasks.
   // TODO: Use ConstantExpr::getNot()?
   if (match(A, (m_Xor(m_SExt(m_Value(Cond)), m_Constant(AConst)))) &&
       match(B, (m_Xor(m_SExt(m_Specific(Cond)), m_Constant(BConst)))) &&
       Cond->getType()->isIntOrIntVectorTy(1) &&
       areInverseVectorBitmasks(AConst, BConst)) {
     AConst = ConstantExpr::getTrunc(AConst, CmpInst::makeCmpResultType(Ty));
     return Builder.CreateXor(Cond, AConst);
   }
   return nullptr;
 }
 
 /// We have an expression of the form (A & C) | (B & D). Try to simplify this
 /// to "A' ? C : D", where A' is a boolean or vector of booleans.
 Value *InstCombinerImpl::matchSelectFromAndOr(Value *A, Value *C, Value *B,
                                               Value *D) {
   // The potential condition of the select may be bitcasted. In that case, look
   // through its bitcast and the corresponding bitcast of the 'not' condition.
   Type *OrigType = A->getType();
   A = peekThroughBitcast(A, true);
   B = peekThroughBitcast(B, true);
   if (Value *Cond = getSelectCondition(A, B)) {
     // ((bc Cond) & C) | ((bc ~Cond) & D) --> bc (select Cond, (bc C), (bc D))
     // If this is a vector, we may need to cast to match the condition's length.
     // The bitcasts will either all exist or all not exist. The builder will
     // not create unnecessary casts if the types already match.
     Type *SelTy = A->getType();
     if (auto *VecTy = dyn_cast<VectorType>(Cond->getType())) {
+      // For a fixed or scalable vector get N from <{vscale x} N x iM>
       unsigned Elts = VecTy->getElementCount().getKnownMinValue();
-      Type *EltTy = Builder.getIntNTy(SelTy->getPrimitiveSizeInBits() / Elts);
+      // For a fixed or scalable vector, get the size in bits of N x iM; for a
+      // scalar this is just M.
+      unsigned SelEltSize = SelTy->getPrimitiveSizeInBits().getKnownMinSize();
+      Type *EltTy = Builder.getIntNTy(SelEltSize / Elts);
       SelTy = VectorType::get(EltTy, VecTy->getElementCount());
     }
     Value *BitcastC = Builder.CreateBitCast(C, SelTy);
     Value *BitcastD = Builder.CreateBitCast(D, SelTy);
     Value *Select = Builder.CreateSelect(Cond, BitcastC, BitcastD);
     return Builder.CreateBitCast(Select, OrigType);
   }
 
   return nullptr;
 }
 
 /// Fold (icmp)|(icmp) if possible.
 Value *InstCombinerImpl::foldOrOfICmps(ICmpInst *LHS, ICmpInst *RHS,
                                        BinaryOperator &Or) {
   const SimplifyQuery Q = SQ.getWithInstruction(&Or);
 
   // Fold (iszero(A & K1) | iszero(A & K2)) ->  (A & (K1 | K2)) != (K1 | K2)
   // if K1 and K2 are a one-bit mask.
   if (Value *V = foldAndOrOfICmpsOfAndWithPow2(LHS, RHS, &Or,
                                                /* IsAnd */ false))
     return V;
 
   ICmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
   Value *LHS0 = LHS->getOperand(0), *RHS0 = RHS->getOperand(0);
   Value *LHS1 = LHS->getOperand(1), *RHS1 = RHS->getOperand(1);
   const APInt *LHSC = nullptr, *RHSC = nullptr;
   match(LHS1, m_APInt(LHSC));
   match(RHS1, m_APInt(RHSC));
 
   // Fold (icmp ult/ule (A + C1), C3) | (icmp ult/ule (A + C2), C3)
   //                   -->  (icmp ult/ule ((A & ~(C1 ^ C2)) + max(C1, C2)), C3)
   // The original condition actually refers to the following two ranges:
   // [MAX_UINT-C1+1, MAX_UINT-C1+1+C3] and [MAX_UINT-C2+1, MAX_UINT-C2+1+C3]
   // We can fold these two ranges if:
   // 1) C1 and C2 is unsigned greater than C3.
   // 2) The two ranges are separated.
   // 3) C1 ^ C2 is one-bit mask.
   // 4) LowRange1 ^ LowRange2 and HighRange1 ^ HighRange2 are one-bit mask.
   // This implies all values in the two ranges differ by exactly one bit.
   if ((PredL == ICmpInst::ICMP_ULT || PredL == ICmpInst::ICMP_ULE) &&
       PredL == PredR && LHSC && RHSC && LHS->hasOneUse() && RHS->hasOneUse() &&
       LHSC->getBitWidth() == RHSC->getBitWidth() && *LHSC == *RHSC) {
 
     Value *AddOpnd;
     const APInt *LAddC, *RAddC;
     if (match(LHS0, m_Add(m_Value(AddOpnd), m_APInt(LAddC))) &&
         match(RHS0, m_Add(m_Specific(AddOpnd), m_APInt(RAddC))) &&
         LAddC->ugt(*LHSC) && RAddC->ugt(*LHSC)) {
 
       APInt DiffC = *LAddC ^ *RAddC;
       if (DiffC.isPowerOf2()) {
         const APInt *MaxAddC = nullptr;
         if (LAddC->ult(*RAddC))
           MaxAddC = RAddC;
         else
           MaxAddC = LAddC;
 
         APInt RRangeLow = -*RAddC;
         APInt RRangeHigh = RRangeLow + *LHSC;
         APInt LRangeLow = -*LAddC;
         APInt LRangeHigh = LRangeLow + *LHSC;
         APInt LowRangeDiff = RRangeLow ^ LRangeLow;
         APInt HighRangeDiff = RRangeHigh ^ LRangeHigh;
         APInt RangeDiff = LRangeLow.sgt(RRangeLow) ? LRangeLow - RRangeLow
                                                    : RRangeLow - LRangeLow;
 
         if (LowRangeDiff.isPowerOf2() && LowRangeDiff == HighRangeDiff &&
             RangeDiff.ugt(*LHSC)) {
           Type *Ty = AddOpnd->getType();
           Value *MaskC = ConstantInt::get(Ty, ~DiffC);
 
           Value *NewAnd = Builder.CreateAnd(AddOpnd, MaskC);
           Value *NewAdd = Builder.CreateAdd(NewAnd,
                                             ConstantInt::get(Ty, *MaxAddC));
           return Builder.CreateICmp(LHS->getPredicate(), NewAdd,
                                     ConstantInt::get(Ty, *LHSC));
         }
       }
     }
   }
 
   // (icmp1 A, B) | (icmp2 A, B) --> (icmp3 A, B)
   if (predicatesFoldable(PredL, PredR)) {
     if (LHS0 == RHS1 && LHS1 == RHS0)
       LHS->swapOperands();
     if (LHS0 == RHS0 && LHS1 == RHS1) {
       unsigned Code = getICmpCode(LHS) | getICmpCode(RHS);
       bool IsSigned = LHS->isSigned() || RHS->isSigned();
       return getNewICmpValue(Code, IsSigned, LHS0, LHS1, Builder);
     }
   }
 
   // handle (roughly):
   // (icmp ne (A & B), C) | (icmp ne (A & D), E)
   if (Value *V = foldLogOpOfMaskedICmps(LHS, RHS, false, Builder))
     return V;
 
   if (LHS->hasOneUse() || RHS->hasOneUse()) {
     // (icmp eq B, 0) | (icmp ult A, B) -> (icmp ule A, B-1)
     // (icmp eq B, 0) | (icmp ugt B, A) -> (icmp ule A, B-1)
     Value *A = nullptr, *B = nullptr;
     if (PredL == ICmpInst::ICMP_EQ && match(LHS1, m_Zero())) {
       B = LHS0;
       if (PredR == ICmpInst::ICMP_ULT && LHS0 == RHS1)
         A = RHS0;
       else if (PredR == ICmpInst::ICMP_UGT && LHS0 == RHS0)
         A = RHS1;
     }
     // (icmp ult A, B) | (icmp eq B, 0) -> (icmp ule A, B-1)
     // (icmp ugt B, A) | (icmp eq B, 0) -> (icmp ule A, B-1)
     else if (PredR == ICmpInst::ICMP_EQ && match(RHS1, m_Zero())) {
       B = RHS0;
       if (PredL == ICmpInst::ICMP_ULT && RHS0 == LHS1)
         A = LHS0;
       else if (PredL == ICmpInst::ICMP_UGT && RHS0 == LHS0)
         A = LHS1;
     }
     if (A && B && B->getType()->isIntOrIntVectorTy())
       return Builder.CreateICmp(
           ICmpInst::ICMP_UGE,
           Builder.CreateAdd(B, Constant::getAllOnesValue(B->getType())), A);
   }
 
   if (Value *V = foldAndOrOfICmpsWithConstEq(LHS, RHS, Or, Builder, Q))
     return V;
   if (Value *V = foldAndOrOfICmpsWithConstEq(RHS, LHS, Or, Builder, Q))
     return V;
 
   // E.g. (icmp slt x, 0) | (icmp sgt x, n) --> icmp ugt x, n
   if (Value *V = simplifyRangeCheck(LHS, RHS, /*Inverted=*/true))
     return V;
 
   // E.g. (icmp sgt x, n) | (icmp slt x, 0) --> icmp ugt x, n
   if (Value *V = simplifyRangeCheck(RHS, LHS, /*Inverted=*/true))
     return V;
 
   if (Value *V = foldAndOrOfEqualityCmpsWithConstants(LHS, RHS, false, Builder))
     return V;
 
   if (Value *V = foldIsPowerOf2(LHS, RHS, false /* JoinedByAnd */, Builder))
     return V;
 
   if (Value *X =
           foldUnsignedUnderflowCheck(LHS, RHS, /*IsAnd=*/false, Q, Builder))
     return X;
   if (Value *X =
           foldUnsignedUnderflowCheck(RHS, LHS, /*IsAnd=*/false, Q, Builder))
     return X;
 
   if (Value *X = foldEqOfParts(LHS, RHS, /*IsAnd=*/false))
     return X;
 
   // (icmp ne A, 0) | (icmp ne B, 0) --> (icmp ne (A|B), 0)
   // TODO: Remove this when foldLogOpOfMaskedICmps can handle undefs.
   if (PredL == ICmpInst::ICMP_NE && match(LHS1, m_ZeroInt()) &&
       PredR == ICmpInst::ICMP_NE && match(RHS1, m_ZeroInt()) &&
       LHS0->getType() == RHS0->getType()) {
     Value *NewOr = Builder.CreateOr(LHS0, RHS0);
     return Builder.CreateICmp(PredL, NewOr,
                               Constant::getNullValue(NewOr->getType()));
   }
 
   // This only handles icmp of constants: (icmp1 A, C1) | (icmp2 B, C2).
   if (!LHSC || !RHSC)
     return nullptr;
 
   return foldAndOrOfICmpsUsingRanges(PredL, LHS0, *LHSC, PredR, RHS0, *RHSC,
                                      Builder, /* IsAnd */ false);
 }
 
 // FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
 // here. We should standardize that construct where it is needed or choose some
 // other way to ensure that commutated variants of patterns are not missed.
 Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
   if (Value *V = SimplifyOrInst(I.getOperand(0), I.getOperand(1),
                                 SQ.getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
 
   if (SimplifyAssociativeOrCommutative(I))
     return &I;
 
   if (Instruction *X = foldVectorBinop(I))
     return X;
 
   if (Instruction *Phi = foldBinopWithPhiOperands(I))
     return Phi;
 
   // See if we can simplify any instructions used by the instruction whose sole
   // purpose is to compute bits we don't care about.
   if (SimplifyDemandedInstructionBits(I))
     return &I;
 
   // Do this before using distributive laws to catch simple and/or/not patterns.
   if (Instruction *Xor = foldOrToXor(I, Builder))
     return Xor;
 
   if (Instruction *X = foldComplexAndOrPatterns(I, Builder))
     return X;
 
   // (A&B)|(A&C) -> A&(B|C) etc
   if (Value *V = SimplifyUsingDistributiveLaws(I))
     return replaceInstUsesWith(I, V);
 
   if (Value *V = SimplifyBSwap(I, Builder))
     return replaceInstUsesWith(I, V);
 
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
   Type *Ty = I.getType();
   if (Ty->isIntOrIntVectorTy(1)) {
     if (auto *SI0 = dyn_cast<SelectInst>(Op0)) {
       if (auto *I =
               foldAndOrOfSelectUsingImpliedCond(Op1, *SI0, /* IsAnd */ false))
         return I;
     }
     if (auto *SI1 = dyn_cast<SelectInst>(Op1)) {
       if (auto *I =
               foldAndOrOfSelectUsingImpliedCond(Op0, *SI1, /* IsAnd */ false))
         return I;
     }
   }
 
   if (Instruction *FoldedLogic = foldBinOpIntoSelectOrPhi(I))
     return FoldedLogic;
 
   if (Instruction *BitOp = matchBSwapOrBitReverse(I, /*MatchBSwaps*/ true,
                                                   /*MatchBitReversals*/ true))
     return BitOp;
 
   if (Instruction *Funnel = matchFunnelShift(I, *this))
     return Funnel;
 
   if (Instruction *Concat = matchOrConcat(I, Builder))
     return replaceInstUsesWith(I, Concat);
 
   Value *X, *Y;
   const APInt *CV;
   if (match(&I, m_c_Or(m_OneUse(m_Xor(m_Value(X), m_APInt(CV))), m_Value(Y))) &&
       !CV->isAllOnes() && MaskedValueIsZero(Y, *CV, 0, &I)) {
     // (X ^ C) | Y -> (X | Y) ^ C iff Y & C == 0
     // The check for a 'not' op is for efficiency (if Y is known zero --> ~X).
     Value *Or = Builder.CreateOr(X, Y);
     return BinaryOperator::CreateXor(Or, ConstantInt::get(Ty, *CV));
   }
 
   // If the operands have no common bits set:
   // or (mul X, Y), X --> add (mul X, Y), X --> mul X, (Y + 1)
   if (match(&I,
             m_c_Or(m_OneUse(m_Mul(m_Value(X), m_Value(Y))), m_Deferred(X))) &&
       haveNoCommonBitsSet(Op0, Op1, DL)) {
     Value *IncrementY = Builder.CreateAdd(Y, ConstantInt::get(Ty, 1));
     return BinaryOperator::CreateMul(X, IncrementY);
   }
 
   // (A & C) | (B & D)
   Value *A, *B, *C, *D;
   if (match(Op0, m_And(m_Value(A), m_Value(C))) &&
       match(Op1, m_And(m_Value(B), m_Value(D)))) {
 
     // (A & C0) | (B & C1)
     const APInt *C0, *C1;
     if (match(C, m_APInt(C0)) && match(D, m_APInt(C1))) {
       Value *X;
       if (*C0 == ~*C1) {
         // ((X | B) & MaskC) | (B & ~MaskC) -> (X & MaskC) | B
         if (match(A, m_c_Or(m_Value(X), m_Specific(B))))
           return BinaryOperator::CreateOr(Builder.CreateAnd(X, *C0), B);
         // (A & MaskC) | ((X | A) & ~MaskC) -> (X & ~MaskC) | A
         if (match(B, m_c_Or(m_Specific(A), m_Value(X))))
           return BinaryOperator::CreateOr(Builder.CreateAnd(X, *C1), A);
 
         // ((X ^ B) & MaskC) | (B & ~MaskC) -> (X & MaskC) ^ B
         if (match(A, m_c_Xor(m_Value(X), m_Specific(B))))
           return BinaryOperator::CreateXor(Builder.CreateAnd(X, *C0), B);
         // (A & MaskC) | ((X ^ A) & ~MaskC) -> (X & ~MaskC) ^ A
         if (match(B, m_c_Xor(m_Specific(A), m_Value(X))))
           return BinaryOperator::CreateXor(Builder.CreateAnd(X, *C1), A);
       }
 
       if ((*C0 & *C1).isZero()) {
         // ((X | B) & C0) | (B & C1) --> (X | B) & (C0 | C1)
         // iff (C0 & C1) == 0 and (X & ~C0) == 0
         if (match(A, m_c_Or(m_Value(X), m_Specific(B))) &&
             MaskedValueIsZero(X, ~*C0, 0, &I)) {
           Constant *C01 = ConstantInt::get(Ty, *C0 | *C1);
           return BinaryOperator::CreateAnd(A, C01);
         }
         // (A & C0) | ((X | A) & C1) --> (X | A) & (C0 | C1)
         // iff (C0 & C1) == 0 and (X & ~C1) == 0
         if (match(B, m_c_Or(m_Value(X), m_Specific(A))) &&
             MaskedValueIsZero(X, ~*C1, 0, &I)) {
           Constant *C01 = ConstantInt::get(Ty, *C0 | *C1);
           return BinaryOperator::CreateAnd(B, C01);
         }
         // ((X | C2) & C0) | ((X | C3) & C1) --> (X | C2 | C3) & (C0 | C1)
         // iff (C0 & C1) == 0 and (C2 & ~C0) == 0 and (C3 & ~C1) == 0.
         const APInt *C2, *C3;
         if (match(A, m_Or(m_Value(X), m_APInt(C2))) &&
             match(B, m_Or(m_Specific(X), m_APInt(C3))) &&
             (*C2 & ~*C0).isZero() && (*C3 & ~*C1).isZero()) {
           Value *Or = Builder.CreateOr(X, *C2 | *C3, "bitfield");
           Constant *C01 = ConstantInt::get(Ty, *C0 | *C1);
           return BinaryOperator::CreateAnd(Or, C01);
         }
       }
     }
 
     // Don't try to form a select if it's unlikely that we'll get rid of at
     // least one of the operands. A select is generally more expensive than the
     // 'or' that it is replacing.
     if (Op0->hasOneUse() || Op1->hasOneUse()) {
       // (Cond & C) | (~Cond & D) -> Cond ? C : D, and commuted variants.
       if (Value *V = matchSelectFromAndOr(A, C, B, D))
         return replaceInstUsesWith(I, V);
       if (Value *V = matchSelectFromAndOr(A, C, D, B))
         return replaceInstUsesWith(I, V);
       if (Value *V = matchSelectFromAndOr(C, A, B, D))
         return replaceInstUsesWith(I, V);
       if (Value *V = matchSelectFromAndOr(C, A, D, B))
         return replaceInstUsesWith(I, V);
       if (Value *V = matchSelectFromAndOr(B, D, A, C))
         return replaceInstUsesWith(I, V);
       if (Value *V = matchSelectFromAndOr(B, D, C, A))
         return replaceInstUsesWith(I, V);
       if (Value *V = matchSelectFromAndOr(D, B, A, C))
         return replaceInstUsesWith(I, V);
       if (Value *V = matchSelectFromAndOr(D, B, C, A))
         return replaceInstUsesWith(I, V);
     }
   }
 
   // (A ^ B) | ((B ^ C) ^ A) -> (A ^ B) | C
   if (match(Op0, m_Xor(m_Value(A), m_Value(B))))
     if (match(Op1, m_Xor(m_Xor(m_Specific(B), m_Value(C)), m_Specific(A))))
       return BinaryOperator::CreateOr(Op0, C);
 
   // ((A ^ C) ^ B) | (B ^ A) -> (B ^ A) | C
   if (match(Op0, m_Xor(m_Xor(m_Value(A), m_Value(C)), m_Value(B))))
     if (match(Op1, m_Xor(m_Specific(B), m_Specific(A))))
       return BinaryOperator::CreateOr(Op1, C);
 
   // ((B | C) & A) | B -> B | (A & C)
   if (match(Op0, m_And(m_Or(m_Specific(Op1), m_Value(C)), m_Value(A))))
     return BinaryOperator::CreateOr(Op1, Builder.CreateAnd(A, C));
 
   if (Instruction *DeMorgan = matchDeMorgansLaws(I, Builder))
     return DeMorgan;
 
   // Canonicalize xor to the RHS.
   bool SwappedForXor = false;
   if (match(Op0, m_Xor(m_Value(), m_Value()))) {
     std::swap(Op0, Op1);
     SwappedForXor = true;
   }
 
   // A | ( A ^ B) -> A |  B
   // A | (~A ^ B) -> A | ~B
   // (A & B) | (A ^ B)
   // ~A | (A ^ B) -> ~(A & B)
   // The swap above should always make Op0 the 'not' for the last case.
   if (match(Op1, m_Xor(m_Value(A), m_Value(B)))) {
     if (Op0 == A || Op0 == B)
       return BinaryOperator::CreateOr(A, B);
 
     if (match(Op0, m_And(m_Specific(A), m_Specific(B))) ||
         match(Op0, m_And(m_Specific(B), m_Specific(A))))
       return BinaryOperator::CreateOr(A, B);
 
     if ((Op0->hasOneUse() || Op1->hasOneUse()) &&
         (match(Op0, m_Not(m_Specific(A))) || match(Op0, m_Not(m_Specific(B)))))
       return BinaryOperator::CreateNot(Builder.CreateAnd(A, B));
 
     if (Op1->hasOneUse() && match(A, m_Not(m_Specific(Op0)))) {
       Value *Not = Builder.CreateNot(B, B->getName() + ".not");
       return BinaryOperator::CreateOr(Not, Op0);
     }
     if (Op1->hasOneUse() && match(B, m_Not(m_Specific(Op0)))) {
       Value *Not = Builder.CreateNot(A, A->getName() + ".not");
       return BinaryOperator::CreateOr(Not, Op0);
     }
   }
 
   // A | ~(A | B) -> A | ~B
   // A | ~(A ^ B) -> A | ~B
   if (match(Op1, m_Not(m_Value(A))))
     if (BinaryOperator *B = dyn_cast<BinaryOperator>(A))
       if ((Op0 == B->getOperand(0) || Op0 == B->getOperand(1)) &&
           Op1->hasOneUse() && (B->getOpcode() == Instruction::Or ||
                                B->getOpcode() == Instruction::Xor)) {
         Value *NotOp = Op0 == B->getOperand(0) ? B->getOperand(1) :
                                                  B->getOperand(0);
         Value *Not = Builder.CreateNot(NotOp, NotOp->getName() + ".not");
         return BinaryOperator::CreateOr(Not, Op0);
       }
 
   if (SwappedForXor)
     std::swap(Op0, Op1);
 
   {
     ICmpInst *LHS = dyn_cast<ICmpInst>(Op0);
     ICmpInst *RHS = dyn_cast<ICmpInst>(Op1);
     if (LHS && RHS)
       if (Value *Res = foldOrOfICmps(LHS, RHS, I))
         return replaceInstUsesWith(I, Res);
 
     // TODO: Make this recursive; it's a little tricky because an arbitrary
     // number of 'or' instructions might have to be created.
     Value *X, *Y;
     if (LHS && match(Op1, m_OneUse(m_Or(m_Value(X), m_Value(Y))))) {
       if (auto *Cmp = dyn_cast<ICmpInst>(X))
         if (Value *Res = foldOrOfICmps(LHS, Cmp, I))
           return replaceInstUsesWith(I, Builder.CreateOr(Res, Y));
       if (auto *Cmp = dyn_cast<ICmpInst>(Y))
         if (Value *Res = foldOrOfICmps(LHS, Cmp, I))
           return replaceInstUsesWith(I, Builder.CreateOr(Res, X));
     }
     if (RHS && match(Op0, m_OneUse(m_Or(m_Value(X), m_Value(Y))))) {
       if (auto *Cmp = dyn_cast<ICmpInst>(X))
         if (Value *Res = foldOrOfICmps(Cmp, RHS, I))
           return replaceInstUsesWith(I, Builder.CreateOr(Res, Y));
       if (auto *Cmp = dyn_cast<ICmpInst>(Y))
         if (Value *Res = foldOrOfICmps(Cmp, RHS, I))
           return replaceInstUsesWith(I, Builder.CreateOr(Res, X));
     }
   }
 
   if (FCmpInst *LHS = dyn_cast<FCmpInst>(I.getOperand(0)))
     if (FCmpInst *RHS = dyn_cast<FCmpInst>(I.getOperand(1)))
       if (Value *Res = foldLogicOfFCmps(LHS, RHS, false))
         return replaceInstUsesWith(I, Res);
 
   if (Instruction *FoldedFCmps = reassociateFCmps(I, Builder))
     return FoldedFCmps;
 
   if (Instruction *CastedOr = foldCastedBitwiseLogic(I))
     return CastedOr;
 
   if (Instruction *Sel = foldBinopOfSextBoolToSelect(I))
     return Sel;
 
   // or(sext(A), B) / or(B, sext(A)) --> A ? -1 : B, where A is i1 or <N x i1>.
   // TODO: Move this into foldBinopOfSextBoolToSelect as a more generalized fold
   //       with binop identity constant. But creating a select with non-constant
   //       arm may not be reversible due to poison semantics. Is that a good
   //       canonicalization?
   if (match(Op0, m_OneUse(m_SExt(m_Value(A)))) &&
       A->getType()->isIntOrIntVectorTy(1))
     return SelectInst::Create(A, ConstantInt::getAllOnesValue(Ty), Op1);
   if (match(Op1, m_OneUse(m_SExt(m_Value(A)))) &&
       A->getType()->isIntOrIntVectorTy(1))
     return SelectInst::Create(A, ConstantInt::getAllOnesValue(Ty), Op0);
 
   // Note: If we've gotten to the point of visiting the outer OR, then the
   // inner one couldn't be simplified.  If it was a constant, then it won't
   // be simplified by a later pass either, so we try swapping the inner/outer
   // ORs in the hopes that we'll be able to simplify it this way.
   // (X|C) | V --> (X|V) | C
   ConstantInt *CI;
   if (Op0->hasOneUse() && !match(Op1, m_ConstantInt()) &&
       match(Op0, m_Or(m_Value(A), m_ConstantInt(CI)))) {
     Value *Inner = Builder.CreateOr(A, Op1);
     Inner->takeName(Op0);
     return BinaryOperator::CreateOr(Inner, CI);
   }
 
   // Change (or (bool?A:B),(bool?C:D)) --> (bool?(or A,C):(or B,D))
   // Since this OR statement hasn't been optimized further yet, we hope
   // that this transformation will allow the new ORs to be optimized.
   {
     Value *X = nullptr, *Y = nullptr;
     if (Op0->hasOneUse() && Op1->hasOneUse() &&
         match(Op0, m_Select(m_Value(X), m_Value(A), m_Value(B))) &&
         match(Op1, m_Select(m_Value(Y), m_Value(C), m_Value(D))) && X == Y) {
       Value *orTrue = Builder.CreateOr(A, C);
       Value *orFalse = Builder.CreateOr(B, D);
       return SelectInst::Create(X, orTrue, orFalse);
     }
   }
 
   // or(ashr(subNSW(Y, X), ScalarSizeInBits(Y) - 1), X)  --> X s> Y ? -1 : X.
   {
     Value *X, *Y;
     if (match(&I, m_c_Or(m_OneUse(m_AShr(
                              m_NSWSub(m_Value(Y), m_Value(X)),
                              m_SpecificInt(Ty->getScalarSizeInBits() - 1))),
                          m_Deferred(X)))) {
       Value *NewICmpInst = Builder.CreateICmpSGT(X, Y);
       Value *AllOnes = ConstantInt::getAllOnesValue(Ty);
       return SelectInst::Create(NewICmpInst, AllOnes, X);
     }
   }
 
   if (Instruction *V =
           canonicalizeCondSignextOfHighBitExtractToSignextHighBitExtract(I))
     return V;
 
   CmpInst::Predicate Pred;
   Value *Mul, *Ov, *MulIsNotZero, *UMulWithOv;
   // Check if the OR weakens the overflow condition for umul.with.overflow by
   // treating any non-zero result as overflow. In that case, we overflow if both
   // umul.with.overflow operands are != 0, as in that case the result can only
   // be 0, iff the multiplication overflows.
   if (match(&I,
             m_c_Or(m_CombineAnd(m_ExtractValue<1>(m_Value(UMulWithOv)),
                                 m_Value(Ov)),
                    m_CombineAnd(m_ICmp(Pred,
                                        m_CombineAnd(m_ExtractValue<0>(
                                                         m_Deferred(UMulWithOv)),
                                                     m_Value(Mul)),
                                        m_ZeroInt()),
                                 m_Value(MulIsNotZero)))) &&
       (Ov->hasOneUse() || (MulIsNotZero->hasOneUse() && Mul->hasOneUse())) &&
       Pred == CmpInst::ICMP_NE) {
     Value *A, *B;
     if (match(UMulWithOv, m_Intrinsic<Intrinsic::umul_with_overflow>(
                               m_Value(A), m_Value(B)))) {
       Value *NotNullA = Builder.CreateIsNotNull(A);
       Value *NotNullB = Builder.CreateIsNotNull(B);
       return BinaryOperator::CreateAnd(NotNullA, NotNullB);
     }
   }
 
   // (~x) | y  -->  ~(x & (~y))  iff that gets rid of inversions
   if (sinkNotIntoOtherHandOfAndOrOr(I))
     return &I;
 
   // Improve "get low bit mask up to and including bit X" pattern:
   //   (1 << X) | ((1 << X) + -1)  -->  -1 l>> (bitwidth(x) - 1 - X)
   if (match(&I, m_c_Or(m_Add(m_Shl(m_One(), m_Value(X)), m_AllOnes()),
                        m_Shl(m_One(), m_Deferred(X)))) &&
       match(&I, m_c_Or(m_OneUse(m_Value()), m_Value()))) {
     Value *Sub = Builder.CreateSub(
         ConstantInt::get(Ty, Ty->getScalarSizeInBits() - 1), X);
     return BinaryOperator::CreateLShr(Constant::getAllOnesValue(Ty), Sub);
   }
 
   // An or recurrence w/loop invariant step is equivelent to (or start, step)
   PHINode *PN = nullptr;
   Value *Start = nullptr, *Step = nullptr;
   if (matchSimpleRecurrence(&I, PN, Start, Step) && DT.dominates(Step, PN))
     return replaceInstUsesWith(I, Builder.CreateOr(Start, Step));
 
   return nullptr;
 }
 
 /// A ^ B can be specified using other logic ops in a variety of patterns. We
 /// can fold these early and efficiently by morphing an existing instruction.
 static Instruction *foldXorToXor(BinaryOperator &I,
                                  InstCombiner::BuilderTy &Builder) {
   assert(I.getOpcode() == Instruction::Xor);
   Value *Op0 = I.getOperand(0);
   Value *Op1 = I.getOperand(1);
   Value *A, *B;
 
   // There are 4 commuted variants for each of the basic patterns.
 
   // (A & B) ^ (A | B) -> A ^ B
   // (A & B) ^ (B | A) -> A ^ B
   // (A | B) ^ (A & B) -> A ^ B
   // (A | B) ^ (B & A) -> A ^ B
   if (match(&I, m_c_Xor(m_And(m_Value(A), m_Value(B)),
                         m_c_Or(m_Deferred(A), m_Deferred(B)))))
     return BinaryOperator::CreateXor(A, B);
 
   // (A | ~B) ^ (~A | B) -> A ^ B
   // (~B | A) ^ (~A | B) -> A ^ B
   // (~A | B) ^ (A | ~B) -> A ^ B
   // (B | ~A) ^ (A | ~B) -> A ^ B
   if (match(&I, m_Xor(m_c_Or(m_Value(A), m_Not(m_Value(B))),
                       m_c_Or(m_Not(m_Deferred(A)), m_Deferred(B)))))
     return BinaryOperator::CreateXor(A, B);
 
   // (A & ~B) ^ (~A & B) -> A ^ B
   // (~B & A) ^ (~A & B) -> A ^ B
   // (~A & B) ^ (A & ~B) -> A ^ B
   // (B & ~A) ^ (A & ~B) -> A ^ B
   if (match(&I, m_Xor(m_c_And(m_Value(A), m_Not(m_Value(B))),
                       m_c_And(m_Not(m_Deferred(A)), m_Deferred(B)))))
     return BinaryOperator::CreateXor(A, B);
 
   // For the remaining cases we need to get rid of one of the operands.
   if (!Op0->hasOneUse() && !Op1->hasOneUse())
     return nullptr;
 
   // (A | B) ^ ~(A & B) -> ~(A ^ B)
   // (A | B) ^ ~(B & A) -> ~(A ^ B)
   // (A & B) ^ ~(A | B) -> ~(A ^ B)
   // (A & B) ^ ~(B | A) -> ~(A ^ B)
   // Complexity sorting ensures the not will be on the right side.
   if ((match(Op0, m_Or(m_Value(A), m_Value(B))) &&
        match(Op1, m_Not(m_c_And(m_Specific(A), m_Specific(B))))) ||
       (match(Op0, m_And(m_Value(A), m_Value(B))) &&
        match(Op1, m_Not(m_c_Or(m_Specific(A), m_Specific(B))))))
     return BinaryOperator::CreateNot(Builder.CreateXor(A, B));
 
   return nullptr;
 }
 
 Value *InstCombinerImpl::foldXorOfICmps(ICmpInst *LHS, ICmpInst *RHS,
                                         BinaryOperator &I) {
   assert(I.getOpcode() == Instruction::Xor && I.getOperand(0) == LHS &&
          I.getOperand(1) == RHS && "Should be 'xor' with these operands");
 
   if (predicatesFoldable(LHS->getPredicate(), RHS->getPredicate())) {
     if (LHS->getOperand(0) == RHS->getOperand(1) &&
         LHS->getOperand(1) == RHS->getOperand(0))
       LHS->swapOperands();
     if (LHS->getOperand(0) == RHS->getOperand(0) &&
         LHS->getOperand(1) == RHS->getOperand(1)) {
       // (icmp1 A, B) ^ (icmp2 A, B) --> (icmp3 A, B)
       Value *Op0 = LHS->getOperand(0), *Op1 = LHS->getOperand(1);
       unsigned Code = getICmpCode(LHS) ^ getICmpCode(RHS);
       bool IsSigned = LHS->isSigned() || RHS->isSigned();
       return getNewICmpValue(Code, IsSigned, Op0, Op1, Builder);
     }
   }
 
   // TODO: This can be generalized to compares of non-signbits using
   // decomposeBitTestICmp(). It could be enhanced more by using (something like)
   // foldLogOpOfMaskedICmps().
   ICmpInst::Predicate PredL = LHS->getPredicate(), PredR = RHS->getPredicate();
   Value *LHS0 = LHS->getOperand(0), *LHS1 = LHS->getOperand(1);
   Value *RHS0 = RHS->getOperand(0), *RHS1 = RHS->getOperand(1);
   if ((LHS->hasOneUse() || RHS->hasOneUse()) &&
       LHS0->getType() == RHS0->getType() &&
       LHS0->getType()->isIntOrIntVectorTy()) {
     // (X > -1) ^ (Y > -1) --> (X ^ Y) < 0
     // (X <  0) ^ (Y <  0) --> (X ^ Y) < 0
     if ((PredL == CmpInst::ICMP_SGT && match(LHS1, m_AllOnes()) &&
          PredR == CmpInst::ICMP_SGT && match(RHS1, m_AllOnes())) ||
         (PredL == CmpInst::ICMP_SLT && match(LHS1, m_Zero()) &&
          PredR == CmpInst::ICMP_SLT && match(RHS1, m_Zero()))) {
       Value *Zero = ConstantInt::getNullValue(LHS0->getType());
       return Builder.CreateICmpSLT(Builder.CreateXor(LHS0, RHS0), Zero);
     }
     // (X > -1) ^ (Y <  0) --> (X ^ Y) > -1
     // (X <  0) ^ (Y > -1) --> (X ^ Y) > -1
     if ((PredL == CmpInst::ICMP_SGT && match(LHS1, m_AllOnes()) &&
          PredR == CmpInst::ICMP_SLT && match(RHS1, m_Zero())) ||
         (PredL == CmpInst::ICMP_SLT && match(LHS1, m_Zero()) &&
          PredR == CmpInst::ICMP_SGT && match(RHS1, m_AllOnes()))) {
       Value *MinusOne = ConstantInt::getAllOnesValue(LHS0->getType());
       return Builder.CreateICmpSGT(Builder.CreateXor(LHS0, RHS0), MinusOne);
     }
   }
 
   // Instead of trying to imitate the folds for and/or, decompose this 'xor'
   // into those logic ops. That is, try to turn this into an and-of-icmps
   // because we have many folds for that pattern.
   //
   // This is based on a truth table definition of xor:
   // X ^ Y --> (X | Y) & !(X & Y)
   if (Value *OrICmp = SimplifyBinOp(Instruction::Or, LHS, RHS, SQ)) {
     // TODO: If OrICmp is true, then the definition of xor simplifies to !(X&Y).
     // TODO: If OrICmp is false, the whole thing is false (InstSimplify?).
     if (Value *AndICmp = SimplifyBinOp(Instruction::And, LHS, RHS, SQ)) {
       // TODO: Independently handle cases where the 'and' side is a constant.
       ICmpInst *X = nullptr, *Y = nullptr;
       if (OrICmp == LHS && AndICmp == RHS) {
         // (LHS | RHS) & !(LHS & RHS) --> LHS & !RHS  --> X & !Y
         X = LHS;
         Y = RHS;
       }
       if (OrICmp == RHS && AndICmp == LHS) {
         // !(LHS & RHS) & (LHS | RHS) --> !LHS & RHS  --> !Y & X
         X = RHS;
         Y = LHS;
       }
       if (X && Y && (Y->hasOneUse() || canFreelyInvertAllUsersOf(Y, &I))) {
         // Invert the predicate of 'Y', thus inverting its output.
         Y->setPredicate(Y->getInversePredicate());
         // So, are there other uses of Y?
         if (!Y->hasOneUse()) {
           // We need to adapt other uses of Y though. Get a value that matches
           // the original value of Y before inversion. While this increases
           // immediate instruction count, we have just ensured that all the
           // users are freely-invertible, so that 'not' *will* get folded away.
           BuilderTy::InsertPointGuard Guard(Builder);
           // Set insertion point to right after the Y.
           Builder.SetInsertPoint(Y->getParent(), ++(Y->getIterator()));
           Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not");
           // Replace all uses of Y (excluding the one in NotY!) with NotY.
           Worklist.pushUsersToWorkList(*Y);
           Y->replaceUsesWithIf(NotY,
                                [NotY](Use &U) { return U.getUser() != NotY; });
         }
         // All done.
         return Builder.CreateAnd(LHS, RHS);
       }
     }
   }
 
   return nullptr;
 }
 
 /// If we have a masked merge, in the canonical form of:
 /// (assuming that A only has one use.)
 ///   |        A  |  |B|
 ///   ((x ^ y) & M) ^ y
 ///    |  D  |
 /// * If M is inverted:
 ///      |  D  |
 ///     ((x ^ y) & ~M) ^ y
 ///   We can canonicalize by swapping the final xor operand
 ///   to eliminate the 'not' of the mask.
 ///     ((x ^ y) & M) ^ x
 /// * If M is a constant, and D has one use, we transform to 'and' / 'or' ops
 ///   because that shortens the dependency chain and improves analysis:
 ///     (x & M) | (y & ~M)
 static Instruction *visitMaskedMerge(BinaryOperator &I,
                                      InstCombiner::BuilderTy &Builder) {
   Value *B, *X, *D;
   Value *M;
   if (!match(&I, m_c_Xor(m_Value(B),
                          m_OneUse(m_c_And(
                              m_CombineAnd(m_c_Xor(m_Deferred(B), m_Value(X)),
                                           m_Value(D)),
                              m_Value(M))))))
     return nullptr;
 
   Value *NotM;
   if (match(M, m_Not(m_Value(NotM)))) {
     // De-invert the mask and swap the value in B part.
     Value *NewA = Builder.CreateAnd(D, NotM);
     return BinaryOperator::CreateXor(NewA, X);
   }
 
   Constant *C;
   if (D->hasOneUse() && match(M, m_Constant(C))) {
     // Propagating undef is unsafe. Clamp undef elements to -1.
     Type *EltTy = C->getType()->getScalarType();
     C = Constant::replaceUndefsWith(C, ConstantInt::getAllOnesValue(EltTy));
     // Unfold.
     Value *LHS = Builder.CreateAnd(X, C);
     Value *NotC = Builder.CreateNot(C);
     Value *RHS = Builder.CreateAnd(B, NotC);
     return BinaryOperator::CreateOr(LHS, RHS);
   }
 
   return nullptr;
 }
 
 // Transform
 //   ~(x ^ y)
 // into:
 //   (~x) ^ y
 // or into
 //   x ^ (~y)
 static Instruction *sinkNotIntoXor(BinaryOperator &I,
                                    InstCombiner::BuilderTy &Builder) {
   Value *X, *Y;
   // FIXME: one-use check is not needed in general, but currently we are unable
   // to fold 'not' into 'icmp', if that 'icmp' has multiple uses. (D35182)
   if (!match(&I, m_Not(m_OneUse(m_Xor(m_Value(X), m_Value(Y))))))
     return nullptr;
 
   // We only want to do the transform if it is free to do.
   if (InstCombiner::isFreeToInvert(X, X->hasOneUse())) {
     // Ok, good.
   } else if (InstCombiner::isFreeToInvert(Y, Y->hasOneUse())) {
     std::swap(X, Y);
   } else
     return nullptr;
 
   Value *NotX = Builder.CreateNot(X, X->getName() + ".not");
   return BinaryOperator::CreateXor(NotX, Y, I.getName() + ".demorgan");
 }
 
 /// Canonicalize a shifty way to code absolute value to the more common pattern
 /// that uses negation and select.
 static Instruction *canonicalizeAbs(BinaryOperator &Xor,
                                     InstCombiner::BuilderTy &Builder) {
   assert(Xor.getOpcode() == Instruction::Xor && "Expected an xor instruction.");
 
   // There are 4 potential commuted variants. Move the 'ashr' candidate to Op1.
   // We're relying on the fact that we only do this transform when the shift has
   // exactly 2 uses and the add has exactly 1 use (otherwise, we might increase
   // instructions).
   Value *Op0 = Xor.getOperand(0), *Op1 = Xor.getOperand(1);
   if (Op0->hasNUses(2))
     std::swap(Op0, Op1);
 
   Type *Ty = Xor.getType();
   Value *A;
   const APInt *ShAmt;
   if (match(Op1, m_AShr(m_Value(A), m_APInt(ShAmt))) &&
       Op1->hasNUses(2) && *ShAmt == Ty->getScalarSizeInBits() - 1 &&
       match(Op0, m_OneUse(m_c_Add(m_Specific(A), m_Specific(Op1))))) {
     // Op1 = ashr i32 A, 31   ; smear the sign bit
     // xor (add A, Op1), Op1  ; add -1 and flip bits if negative
     // --> (A < 0) ? -A : A
     Value *Cmp = Builder.CreateICmpSLT(A, ConstantInt::getNullValue(Ty));
     // Copy the nuw/nsw flags from the add to the negate.
     auto *Add = cast<BinaryOperator>(Op0);
     Value *Neg = Builder.CreateNeg(A, "", Add->hasNoUnsignedWrap(),
                                    Add->hasNoSignedWrap());
     return SelectInst::Create(Cmp, Neg, A);
   }
   return nullptr;
 }
 
 // Transform
 //   z = (~x) &/| y
 // into:
 //   z = ~(x |/& (~y))
 // iff y is free to invert and all uses of z can be freely updated.
 bool InstCombinerImpl::sinkNotIntoOtherHandOfAndOrOr(BinaryOperator &I) {
   Instruction::BinaryOps NewOpc;
   switch (I.getOpcode()) {
   case Instruction::And:
     NewOpc = Instruction::Or;
     break;
   case Instruction::Or:
     NewOpc = Instruction::And;
     break;
   default:
     return false;
   };
 
   Value *X, *Y;
   if (!match(&I, m_c_BinOp(m_Not(m_Value(X)), m_Value(Y))))
     return false;
 
   // Will we be able to fold the `not` into Y eventually?
   if (!InstCombiner::isFreeToInvert(Y, Y->hasOneUse()))
     return false;
 
   // And can our users be adapted?
   if (!InstCombiner::canFreelyInvertAllUsersOf(&I, /*IgnoredUser=*/nullptr))
     return false;
 
   Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not");
   Value *NewBinOp =
       BinaryOperator::Create(NewOpc, X, NotY, I.getName() + ".not");
   Builder.Insert(NewBinOp);
   replaceInstUsesWith(I, NewBinOp);
   // We can not just create an outer `not`, it will most likely be immediately
   // folded back, reconstructing our initial pattern, and causing an
   // infinite combine loop, so immediately manually fold it away.
   freelyInvertAllUsersOf(NewBinOp);
   return true;
 }
 
 Instruction *InstCombinerImpl::foldNot(BinaryOperator &I) {
   Value *NotOp;
   if (!match(&I, m_Not(m_Value(NotOp))))
     return nullptr;
 
   // Apply DeMorgan's Law for 'nand' / 'nor' logic with an inverted operand.
   // We must eliminate the and/or (one-use) for these transforms to not increase
   // the instruction count.
   //
   // ~(~X & Y) --> (X | ~Y)
   // ~(Y & ~X) --> (X | ~Y)
   //
   // Note: The logical matches do not check for the commuted patterns because
   //       those are handled via SimplifySelectsFeedingBinaryOp().
   Type *Ty = I.getType();
   Value *X, *Y;
   if (match(NotOp, m_OneUse(m_c_And(m_Not(m_Value(X)), m_Value(Y))))) {
     Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not");
     return BinaryOperator::CreateOr(X, NotY);
   }
   if (match(NotOp, m_OneUse(m_LogicalAnd(m_Not(m_Value(X)), m_Value(Y))))) {
     Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not");
     return SelectInst::Create(X, ConstantInt::getTrue(Ty), NotY);
   }
 
   // ~(~X | Y) --> (X & ~Y)
   // ~(Y | ~X) --> (X & ~Y)
   if (match(NotOp, m_OneUse(m_c_Or(m_Not(m_Value(X)), m_Value(Y))))) {
     Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not");
     return BinaryOperator::CreateAnd(X, NotY);
   }
   if (match(NotOp, m_OneUse(m_LogicalOr(m_Not(m_Value(X)), m_Value(Y))))) {
     Value *NotY = Builder.CreateNot(Y, Y->getName() + ".not");
     return SelectInst::Create(X, NotY, ConstantInt::getFalse(Ty));
   }
 
   // Is this a 'not' (~) fed by a binary operator?
   BinaryOperator *NotVal;
   if (match(NotOp, m_BinOp(NotVal))) {
     if (NotVal->getOpcode() == Instruction::And ||
         NotVal->getOpcode() == Instruction::Or) {
       // Apply DeMorgan's Law when inverts are free:
       // ~(X & Y) --> (~X | ~Y)
       // ~(X | Y) --> (~X & ~Y)
       if (isFreeToInvert(NotVal->getOperand(0),
                          NotVal->getOperand(0)->hasOneUse()) &&
           isFreeToInvert(NotVal->getOperand(1),
                          NotVal->getOperand(1)->hasOneUse())) {
         Value *NotX = Builder.CreateNot(NotVal->getOperand(0), "notlhs");
         Value *NotY = Builder.CreateNot(NotVal->getOperand(1), "notrhs");
         if (NotVal->getOpcode() == Instruction::And)
           return BinaryOperator::CreateOr(NotX, NotY);
         return BinaryOperator::CreateAnd(NotX, NotY);
       }
     }
 
     // ~((-X) | Y) --> (X - 1) & (~Y)
     if (match(NotVal,
               m_OneUse(m_c_Or(m_OneUse(m_Neg(m_Value(X))), m_Value(Y))))) {
       Value *DecX = Builder.CreateAdd(X, ConstantInt::getAllOnesValue(Ty));
       Value *NotY = Builder.CreateNot(Y);
       return BinaryOperator::CreateAnd(DecX, NotY);
     }
 
     // ~(~X >>s Y) --> (X >>s Y)
     if (match(NotVal, m_AShr(m_Not(m_Value(X)), m_Value(Y))))
       return BinaryOperator::CreateAShr(X, Y);
 
     // If we are inverting a right-shifted constant, we may be able to eliminate
     // the 'not' by inverting the constant and using the opposite shift type.
     // Canonicalization rules ensure that only a negative constant uses 'ashr',
     // but we must check that in case that transform has not fired yet.
 
     // ~(C >>s Y) --> ~C >>u Y (when inverting the replicated sign bits)
     Constant *C;
     if (match(NotVal, m_AShr(m_Constant(C), m_Value(Y))) &&
         match(C, m_Negative())) {
       // We matched a negative constant, so propagating undef is unsafe.
       // Clamp undef elements to -1.
       Type *EltTy = Ty->getScalarType();
       C = Constant::replaceUndefsWith(C, ConstantInt::getAllOnesValue(EltTy));
       return BinaryOperator::CreateLShr(ConstantExpr::getNot(C), Y);
     }
 
     // ~(C >>u Y) --> ~C >>s Y (when inverting the replicated sign bits)
     if (match(NotVal, m_LShr(m_Constant(C), m_Value(Y))) &&
         match(C, m_NonNegative())) {
       // We matched a non-negative constant, so propagating undef is unsafe.
       // Clamp undef elements to 0.
       Type *EltTy = Ty->getScalarType();
       C = Constant::replaceUndefsWith(C, ConstantInt::getNullValue(EltTy));
       return BinaryOperator::CreateAShr(ConstantExpr::getNot(C), Y);
     }
 
     // ~(X + C) --> ~C - X
     if (match(NotVal, m_c_Add(m_Value(X), m_ImmConstant(C))))
       return BinaryOperator::CreateSub(ConstantExpr::getNot(C), X);
 
     // ~(X - Y) --> ~X + Y
     // FIXME: is it really beneficial to sink the `not` here?
     if (match(NotVal, m_Sub(m_Value(X), m_Value(Y))))
       if (isa<Constant>(X) || NotVal->hasOneUse())
         return BinaryOperator::CreateAdd(Builder.CreateNot(X), Y);
 
     // ~(~X + Y) --> X - Y
     if (match(NotVal, m_c_Add(m_Not(m_Value(X)), m_Value(Y))))
       return BinaryOperator::CreateWithCopiedFlags(Instruction::Sub, X, Y,
                                                    NotVal);
   }
 
   // not (cmp A, B) = !cmp A, B
   CmpInst::Predicate Pred;
   if (match(NotOp, m_OneUse(m_Cmp(Pred, m_Value(), m_Value())))) {
     cast<CmpInst>(NotOp)->setPredicate(CmpInst::getInversePredicate(Pred));
     return replaceInstUsesWith(I, NotOp);
   }
 
   // Eliminate a bitwise 'not' op of 'not' min/max by inverting the min/max:
   // ~min(~X, ~Y) --> max(X, Y)
   // ~max(~X, Y) --> min(X, ~Y)
   auto *II = dyn_cast<IntrinsicInst>(NotOp);
   if (II && II->hasOneUse()) {
     if (match(NotOp, m_MaxOrMin(m_Value(X), m_Value(Y))) &&
         isFreeToInvert(X, X->hasOneUse()) &&
         isFreeToInvert(Y, Y->hasOneUse())) {
       Intrinsic::ID InvID = getInverseMinMaxIntrinsic(II->getIntrinsicID());
       Value *NotX = Builder.CreateNot(X);
       Value *NotY = Builder.CreateNot(Y);
       Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, NotX, NotY);
       return replaceInstUsesWith(I, InvMaxMin);
     }
     if (match(NotOp, m_c_MaxOrMin(m_Not(m_Value(X)), m_Value(Y)))) {
       Intrinsic::ID InvID = getInverseMinMaxIntrinsic(II->getIntrinsicID());
       Value *NotY = Builder.CreateNot(Y);
       Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, X, NotY);
       return replaceInstUsesWith(I, InvMaxMin);
     }
   }
 
   // TODO: Remove folds if we canonicalize to intrinsics (see above).
   // Eliminate a bitwise 'not' op of 'not' min/max by inverting the min/max:
   //
   //   %notx = xor i32 %x, -1
   //   %cmp1 = icmp sgt i32 %notx, %y
   //   %smax = select i1 %cmp1, i32 %notx, i32 %y
   //   %res = xor i32 %smax, -1
   // =>
   //   %noty = xor i32 %y, -1
   //   %cmp2 = icmp slt %x, %noty
   //   %res = select i1 %cmp2, i32 %x, i32 %noty
   //
   // Same is applicable for smin/umax/umin.
   if (NotOp->hasOneUse()) {
     Value *LHS, *RHS;
     SelectPatternFlavor SPF = matchSelectPattern(NotOp, LHS, RHS).Flavor;
     if (SelectPatternResult::isMinOrMax(SPF)) {
       // It's possible we get here before the not has been simplified, so make
       // sure the input to the not isn't freely invertible.
       if (match(LHS, m_Not(m_Value(X))) && !isFreeToInvert(X, X->hasOneUse())) {
         Value *NotY = Builder.CreateNot(RHS);
         return SelectInst::Create(
             Builder.CreateICmp(getInverseMinMaxPred(SPF), X, NotY), X, NotY);
       }
 
       // It's possible we get here before the not has been simplified, so make
       // sure the input to the not isn't freely invertible.
       if (match(RHS, m_Not(m_Value(Y))) && !isFreeToInvert(Y, Y->hasOneUse())) {
         Value *NotX = Builder.CreateNot(LHS);
         return SelectInst::Create(
             Builder.CreateICmp(getInverseMinMaxPred(SPF), NotX, Y), NotX, Y);
       }
 
       // If both sides are freely invertible, then we can get rid of the xor
       // completely.
       if (isFreeToInvert(LHS, !LHS->hasNUsesOrMore(3)) &&
           isFreeToInvert(RHS, !RHS->hasNUsesOrMore(3))) {
         Value *NotLHS = Builder.CreateNot(LHS);
         Value *NotRHS = Builder.CreateNot(RHS);
         return SelectInst::Create(
             Builder.CreateICmp(getInverseMinMaxPred(SPF), NotLHS, NotRHS),
             NotLHS, NotRHS);
       }
     }
 
     // Pull 'not' into operands of select if both operands are one-use compares
     // or one is one-use compare and the other one is a constant.
     // Inverting the predicates eliminates the 'not' operation.
     // Example:
     //   not (select ?, (cmp TPred, ?, ?), (cmp FPred, ?, ?) -->
     //     select ?, (cmp InvTPred, ?, ?), (cmp InvFPred, ?, ?)
     //   not (select ?, (cmp TPred, ?, ?), true -->
     //     select ?, (cmp InvTPred, ?, ?), false
     if (auto *Sel = dyn_cast<SelectInst>(NotOp)) {
       Value *TV = Sel->getTrueValue();
       Value *FV = Sel->getFalseValue();
       auto *CmpT = dyn_cast<CmpInst>(TV);
       auto *CmpF = dyn_cast<CmpInst>(FV);
       bool InvertibleT = (CmpT && CmpT->hasOneUse()) || isa<Constant>(TV);
       bool InvertibleF = (CmpF && CmpF->hasOneUse()) || isa<Constant>(FV);
       if (InvertibleT && InvertibleF) {
         if (CmpT)
           CmpT->setPredicate(CmpT->getInversePredicate());
         else
           Sel->setTrueValue(ConstantExpr::getNot(cast<Constant>(TV)));
         if (CmpF)
           CmpF->setPredicate(CmpF->getInversePredicate());
         else
           Sel->setFalseValue(ConstantExpr::getNot(cast<Constant>(FV)));
         return replaceInstUsesWith(I, Sel);
       }
     }
   }
 
   if (Instruction *NewXor = sinkNotIntoXor(I, Builder))
     return NewXor;
 
   return nullptr;
 }
 
 // FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
 // here. We should standardize that construct where it is needed or choose some
 // other way to ensure that commutated variants of patterns are not missed.
 Instruction *InstCombinerImpl::visitXor(BinaryOperator &I) {
   if (Value *V = SimplifyXorInst(I.getOperand(0), I.getOperand(1),
                                  SQ.getWithInstruction(&I)))
     return replaceInstUsesWith(I, V);
 
   if (SimplifyAssociativeOrCommutative(I))
     return &I;
 
   if (Instruction *X = foldVectorBinop(I))
     return X;
 
   if (Instruction *Phi = foldBinopWithPhiOperands(I))
     return Phi;
 
   if (Instruction *NewXor = foldXorToXor(I, Builder))
     return NewXor;
 
   // (A&B)^(A&C) -> A&(B^C) etc
   if (Value *V = SimplifyUsingDistributiveLaws(I))
     return replaceInstUsesWith(I, V);
 
   // See if we can simplify any instructions used by the instruction whose sole
   // purpose is to compute bits we don't care about.
   if (SimplifyDemandedInstructionBits(I))
     return &I;
 
   if (Value *V = SimplifyBSwap(I, Builder))
     return replaceInstUsesWith(I, V);
 
   if (Instruction *R = foldNot(I))
     return R;
 
   // Fold (X & M) ^ (Y & ~M) -> (X & M) | (Y & ~M)
   // This it a special case in haveNoCommonBitsSet, but the computeKnownBits
   // calls in there are unnecessary as SimplifyDemandedInstructionBits should
   // have already taken care of those cases.
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
   Value *M;
   if (match(&I, m_c_Xor(m_c_And(m_Not(m_Value(M)), m_Value()),
                         m_c_And(m_Deferred(M), m_Value()))))
     return BinaryOperator::CreateOr(Op0, Op1);
 
   if (Instruction *Xor = visitMaskedMerge(I, Builder))
     return Xor;
 
   Value *X, *Y;
   Constant *C1;
   if (match(Op1, m_Constant(C1))) {
     // Use DeMorgan and reassociation to eliminate a 'not' op.
     Constant *C2;
     if (match(Op0, m_OneUse(m_Or(m_Not(m_Value(X)), m_Constant(C2))))) {
       // (~X | C2) ^ C1 --> ((X & ~C2) ^ -1) ^ C1 --> (X & ~C2) ^ ~C1
       Value *And = Builder.CreateAnd(X, ConstantExpr::getNot(C2));
       return BinaryOperator::CreateXor(And, ConstantExpr::getNot(C1));
     }
     if (match(Op0, m_OneUse(m_And(m_Not(m_Value(X)), m_Constant(C2))))) {
       // (~X & C2) ^ C1 --> ((X | ~C2) ^ -1) ^ C1 --> (X | ~C2) ^ ~C1
       Value *Or = Builder.CreateOr(X, ConstantExpr::getNot(C2));
       return BinaryOperator::CreateXor(Or, ConstantExpr::getNot(C1));
     }
 
     // Convert xor ([trunc] (ashr X, BW-1)), C =>
     //   select(X >s -1, C, ~C)
     // The ashr creates "AllZeroOrAllOne's", which then optionally inverses the
     // constant depending on whether this input is less than 0.
     const APInt *CA;
     if (match(Op0, m_OneUse(m_TruncOrSelf(
                        m_AShr(m_Value(X), m_APIntAllowUndef(CA))))) &&
         *CA == X->getType()->getScalarSizeInBits() - 1 &&
         !match(C1, m_AllOnes())) {
       assert(!C1->isZeroValue() && "Unexpected xor with 0");
       Value *ICmp =
           Builder.CreateICmpSGT(X, Constant::getAllOnesValue(X->getType()));
       return SelectInst::Create(ICmp, Op1, Builder.CreateNot(Op1));
     }
   }
 
   Type *Ty = I.getType();
   {
     const APInt *RHSC;
     if (match(Op1, m_APInt(RHSC))) {
       Value *X;
       const APInt *C;
       // (C - X) ^ signmaskC --> (C + signmaskC) - X
       if (RHSC->isSignMask() && match(Op0, m_Sub(m_APInt(C), m_Value(X))))
         return BinaryOperator::CreateSub(ConstantInt::get(Ty, *C + *RHSC), X);
 
       // (X + C) ^ signmaskC --> X + (C + signmaskC)
       if (RHSC->isSignMask() && match(Op0, m_Add(m_Value(X), m_APInt(C))))
         return BinaryOperator::CreateAdd(X, ConstantInt::get(Ty, *C + *RHSC));
 
       // (X | C) ^ RHSC --> X ^ (C ^ RHSC) iff X & C == 0
       if (match(Op0, m_Or(m_Value(X), m_APInt(C))) &&
           MaskedValueIsZero(X, *C, 0, &I))
         return BinaryOperator::CreateXor(X, ConstantInt::get(Ty, *C ^ *RHSC));
 
       // If RHSC is inverting the remaining bits of shifted X,
       // canonicalize to a 'not' before the shift to help SCEV and codegen:
       // (X << C) ^ RHSC --> ~X << C
       if (match(Op0, m_OneUse(m_Shl(m_Value(X), m_APInt(C)))) &&
           *RHSC == APInt::getAllOnes(Ty->getScalarSizeInBits()).shl(*C)) {
         Value *NotX = Builder.CreateNot(X);
         return BinaryOperator::CreateShl(NotX, ConstantInt::get(Ty, *C));
       }
       // (X >>u C) ^ RHSC --> ~X >>u C
       if (match(Op0, m_OneUse(m_LShr(m_Value(X), m_APInt(C)))) &&
           *RHSC == APInt::getAllOnes(Ty->getScalarSizeInBits()).lshr(*C)) {
         Value *NotX = Builder.CreateNot(X);
         return BinaryOperator::CreateLShr(NotX, ConstantInt::get(Ty, *C));
       }
       // TODO: We could handle 'ashr' here as well. That would be matching
       //       a 'not' op and moving it before the shift. Doing that requires
       //       preventing the inverse fold in canShiftBinOpWithConstantRHS().
     }
   }
 
   // FIXME: This should not be limited to scalar (pull into APInt match above).
   {
     Value *X;
     ConstantInt *C1, *C2, *C3;
     // ((X^C1) >> C2) ^ C3 -> (X>>C2) ^ ((C1>>C2)^C3)
     if (match(Op1, m_ConstantInt(C3)) &&
         match(Op0, m_LShr(m_Xor(m_Value(X), m_ConstantInt(C1)),
                           m_ConstantInt(C2))) &&
         Op0->hasOneUse()) {
       // fold (C1 >> C2) ^ C3
       APInt FoldConst = C1->getValue().lshr(C2->getValue());
       FoldConst ^= C3->getValue();
       // Prepare the two operands.
       auto *Opnd0 = cast<Instruction>(Builder.CreateLShr(X, C2));
       Opnd0->takeName(cast<Instruction>(Op0));
       Opnd0->setDebugLoc(I.getDebugLoc());
       return BinaryOperator::CreateXor(Opnd0, ConstantInt::get(Ty, FoldConst));
     }
   }
 
   if (Instruction *FoldedLogic = foldBinOpIntoSelectOrPhi(I))
     return FoldedLogic;
 
   // Y ^ (X | Y) --> X & ~Y
   // Y ^ (Y | X) --> X & ~Y
   if (match(Op1, m_OneUse(m_c_Or(m_Value(X), m_Specific(Op0)))))
     return BinaryOperator::CreateAnd(X, Builder.CreateNot(Op0));
   // (X | Y) ^ Y --> X & ~Y
   // (Y | X) ^ Y --> X & ~Y
   if (match(Op0, m_OneUse(m_c_Or(m_Value(X), m_Specific(Op1)))))
     return BinaryOperator::CreateAnd(X, Builder.CreateNot(Op1));
 
   // Y ^ (X & Y) --> ~X & Y
   // Y ^ (Y & X) --> ~X & Y
   if (match(Op1, m_OneUse(m_c_And(m_Value(X), m_Specific(Op0)))))
     return BinaryOperator::CreateAnd(Op0, Builder.CreateNot(X));
   // (X & Y) ^ Y --> ~X & Y
   // (Y & X) ^ Y --> ~X & Y
   // Canonical form is (X & C) ^ C; don't touch that.
   // TODO: A 'not' op is better for analysis and codegen, but demanded bits must
   //       be fixed to prefer that (otherwise we get infinite looping).
   if (!match(Op1, m_Constant()) &&
       match(Op0, m_OneUse(m_c_And(m_Value(X), m_Specific(Op1)))))
     return BinaryOperator::CreateAnd(Op1, Builder.CreateNot(X));
 
   Value *A, *B, *C;
   // (A ^ B) ^ (A | C) --> (~A & C) ^ B -- There are 4 commuted variants.
   if (match(&I, m_c_Xor(m_OneUse(m_Xor(m_Value(A), m_Value(B))),
                         m_OneUse(m_c_Or(m_Deferred(A), m_Value(C))))))
       return BinaryOperator::CreateXor(
           Builder.CreateAnd(Builder.CreateNot(A), C), B);
 
   // (A ^ B) ^ (B | C) --> (~B & C) ^ A -- There are 4 commuted variants.
   if (match(&I, m_c_Xor(m_OneUse(m_Xor(m_Value(A), m_Value(B))),
                         m_OneUse(m_c_Or(m_Deferred(B), m_Value(C))))))
       return BinaryOperator::CreateXor(
           Builder.CreateAnd(Builder.CreateNot(B), C), A);
 
   // (A & B) ^ (A ^ B) -> (A | B)
   if (match(Op0, m_And(m_Value(A), m_Value(B))) &&
       match(Op1, m_c_Xor(m_Specific(A), m_Specific(B))))
     return BinaryOperator::CreateOr(A, B);
   // (A ^ B) ^ (A & B) -> (A | B)
   if (match(Op0, m_Xor(m_Value(A), m_Value(B))) &&
       match(Op1, m_c_And(m_Specific(A), m_Specific(B))))
     return BinaryOperator::CreateOr(A, B);
 
   // (A & ~B) ^ ~A -> ~(A & B)
   // (~B & A) ^ ~A -> ~(A & B)
   if (match(Op0, m_c_And(m_Value(A), m_Not(m_Value(B)))) &&
       match(Op1, m_Not(m_Specific(A))))
     return BinaryOperator::CreateNot(Builder.CreateAnd(A, B));
 
   // (~A & B) ^ A --> A | B -- There are 4 commuted variants.
   if (match(&I, m_c_Xor(m_c_And(m_Not(m_Value(A)), m_Value(B)), m_Deferred(A))))
     return BinaryOperator::CreateOr(A, B);
 
   // (~A | B) ^ A --> ~(A & B)
   if (match(Op0, m_OneUse(m_c_Or(m_Not(m_Specific(Op1)), m_Value(B)))))
     return BinaryOperator::CreateNot(Builder.CreateAnd(Op1, B));
 
   // A ^ (~A | B) --> ~(A & B)
   if (match(Op1, m_OneUse(m_c_Or(m_Not(m_Specific(Op0)), m_Value(B)))))
     return BinaryOperator::CreateNot(Builder.CreateAnd(Op0, B));
 
   // (A | B) ^ (A | C) --> (B ^ C) & ~A -- There are 4 commuted variants.
   // TODO: Loosen one-use restriction if common operand is a constant.
   Value *D;
   if (match(Op0, m_OneUse(m_Or(m_Value(A), m_Value(B)))) &&
       match(Op1, m_OneUse(m_Or(m_Value(C), m_Value(D))))) {
     if (B == C || B == D)
       std::swap(A, B);
     if (A == C)
       std::swap(C, D);
     if (A == D) {
       Value *NotA = Builder.CreateNot(A);
       return BinaryOperator::CreateAnd(Builder.CreateXor(B, C), NotA);
     }
   }
 
   if (auto *LHS = dyn_cast<ICmpInst>(I.getOperand(0)))
     if (auto *RHS = dyn_cast<ICmpInst>(I.getOperand(1)))
       if (Value *V = foldXorOfICmps(LHS, RHS, I))
         return replaceInstUsesWith(I, V);
 
   if (Instruction *CastedXor = foldCastedBitwiseLogic(I))
     return CastedXor;
 
   if (Instruction *Abs = canonicalizeAbs(I, Builder))
     return Abs;
 
   // Otherwise, if all else failed, try to hoist the xor-by-constant:
   //   (X ^ C) ^ Y --> (X ^ Y) ^ C
   // Just like we do in other places, we completely avoid the fold
   // for constantexprs, at least to avoid endless combine loop.
   if (match(&I, m_c_Xor(m_OneUse(m_Xor(m_CombineAnd(m_Value(X),
                                                     m_Unless(m_ConstantExpr())),
                                        m_ImmConstant(C1))),
                         m_Value(Y))))
     return BinaryOperator::CreateXor(Builder.CreateXor(X, Y), C1);
 
   return nullptr;
 }
diff --git a/contrib/llvm-project/llvm/lib/Transforms/Scalar/SCCP.cpp b/contrib/llvm-project/llvm/lib/Transforms/Scalar/SCCP.cpp
index c34da51e6dc1..fa1cfc84e4fd 100644
--- a/contrib/llvm-project/llvm/lib/Transforms/Scalar/SCCP.cpp
+++ b/contrib/llvm-project/llvm/lib/Transforms/Scalar/SCCP.cpp
@@ -1,663 +1,682 @@
 //===- SCCP.cpp - Sparse Conditional Constant Propagation -----------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements sparse conditional constant propagation and merging:
 //
 // Specifically, this:
 //   * Assumes values are constant unless proven otherwise
 //   * Assumes BasicBlocks are dead unless proven otherwise
 //   * Proves values to be constant, and replaces them with constants
 //   * Proves conditional branches to be unconditional
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Scalar/SCCP.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueLattice.h"
 #include "llvm/Analysis/ValueLatticeUtils.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/InstVisitor.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/User.h"
 #include "llvm/IR/Value.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/PredicateInfo.h"
 #include <cassert>
 #include <utility>
 #include <vector>
 
 using namespace llvm;
 
 #define DEBUG_TYPE "sccp"
 
 STATISTIC(NumInstRemoved, "Number of instructions removed");
 STATISTIC(NumDeadBlocks , "Number of basic blocks unreachable");
 STATISTIC(NumInstReplaced,
           "Number of instructions replaced with (simpler) instruction");
 
 STATISTIC(IPNumInstRemoved, "Number of instructions removed by IPSCCP");
 STATISTIC(IPNumArgsElimed ,"Number of arguments constant propagated by IPSCCP");
 STATISTIC(IPNumGlobalConst, "Number of globals found to be constant by IPSCCP");
 STATISTIC(
     IPNumInstReplaced,
     "Number of instructions replaced with (simpler) instruction by IPSCCP");
 
 // Helper to check if \p LV is either a constant or a constant
 // range with a single element. This should cover exactly the same cases as the
 // old ValueLatticeElement::isConstant() and is intended to be used in the
 // transition to ValueLatticeElement.
 static bool isConstant(const ValueLatticeElement &LV) {
   return LV.isConstant() ||
          (LV.isConstantRange() && LV.getConstantRange().isSingleElement());
 }
 
 // Helper to check if \p LV is either overdefined or a constant range with more
 // than a single element. This should cover exactly the same cases as the old
 // ValueLatticeElement::isOverdefined() and is intended to be used in the
 // transition to ValueLatticeElement.
 static bool isOverdefined(const ValueLatticeElement &LV) {
   return !LV.isUnknownOrUndef() && !isConstant(LV);
 }
 
 static bool tryToReplaceWithConstant(SCCPSolver &Solver, Value *V) {
   Constant *Const = nullptr;
   if (V->getType()->isStructTy()) {
     std::vector<ValueLatticeElement> IVs = Solver.getStructLatticeValueFor(V);
     if (llvm::any_of(IVs, isOverdefined))
       return false;
     std::vector<Constant *> ConstVals;
     auto *ST = cast<StructType>(V->getType());
     for (unsigned i = 0, e = ST->getNumElements(); i != e; ++i) {
       ValueLatticeElement V = IVs[i];
       ConstVals.push_back(isConstant(V)
                               ? Solver.getConstant(V)
                               : UndefValue::get(ST->getElementType(i)));
     }
     Const = ConstantStruct::get(ST, ConstVals);
   } else {
     const ValueLatticeElement &IV = Solver.getLatticeValueFor(V);
     if (isOverdefined(IV))
       return false;
 
     Const =
         isConstant(IV) ? Solver.getConstant(IV) : UndefValue::get(V->getType());
   }
   assert(Const && "Constant is nullptr here!");
 
   // Replacing `musttail` instructions with constant breaks `musttail` invariant
   // unless the call itself can be removed.
   // Calls with "clang.arc.attachedcall" implicitly use the return value and
   // those uses cannot be updated with a constant.
   CallBase *CB = dyn_cast<CallBase>(V);
   if (CB && ((CB->isMustTailCall() && !CB->isSafeToRemove()) ||
              CB->getOperandBundle(LLVMContext::OB_clang_arc_attachedcall))) {
     Function *F = CB->getCalledFunction();
 
     // Don't zap returns of the callee
     if (F)
       Solver.addToMustPreserveReturnsInFunctions(F);
 
     LLVM_DEBUG(dbgs() << "  Can\'t treat the result of call " << *CB
                       << " as a constant\n");
     return false;
   }
 
   LLVM_DEBUG(dbgs() << "  Constant: " << *Const << " = " << *V << '\n');
 
   // Replaces all of the uses of a variable with uses of the constant.
   V->replaceAllUsesWith(Const);
   return true;
 }
 
 static bool simplifyInstsInBlock(SCCPSolver &Solver, BasicBlock &BB,
                                  SmallPtrSetImpl<Value *> &InsertedValues,
                                  Statistic &InstRemovedStat,
                                  Statistic &InstReplacedStat) {
   bool MadeChanges = false;
   for (Instruction &Inst : make_early_inc_range(BB)) {
     if (Inst.getType()->isVoidTy())
       continue;
     if (tryToReplaceWithConstant(Solver, &Inst)) {
       if (Inst.isSafeToRemove())
         Inst.eraseFromParent();
 
       MadeChanges = true;
       ++InstRemovedStat;
     } else if (isa<SExtInst>(&Inst)) {
       Value *ExtOp = Inst.getOperand(0);
       if (isa<Constant>(ExtOp) || InsertedValues.count(ExtOp))
         continue;
       const ValueLatticeElement &IV = Solver.getLatticeValueFor(ExtOp);
       if (!IV.isConstantRange(/*UndefAllowed=*/false))
         continue;
       if (IV.getConstantRange().isAllNonNegative()) {
         auto *ZExt = new ZExtInst(ExtOp, Inst.getType(), "", &Inst);
         InsertedValues.insert(ZExt);
         Inst.replaceAllUsesWith(ZExt);
         Solver.removeLatticeValueFor(&Inst);
         Inst.eraseFromParent();
         InstReplacedStat++;
         MadeChanges = true;
       }
     }
   }
   return MadeChanges;
 }
 
 // runSCCP() - Run the Sparse Conditional Constant Propagation algorithm,
 // and return true if the function was modified.
 static bool runSCCP(Function &F, const DataLayout &DL,
                     const TargetLibraryInfo *TLI) {
   LLVM_DEBUG(dbgs() << "SCCP on function '" << F.getName() << "'\n");
   SCCPSolver Solver(
       DL, [TLI](Function &F) -> const TargetLibraryInfo & { return *TLI; },
       F.getContext());
 
   // Mark the first block of the function as being executable.
   Solver.markBlockExecutable(&F.front());
 
   // Mark all arguments to the function as being overdefined.
   for (Argument &AI : F.args())
     Solver.markOverdefined(&AI);
 
   // Solve for constants.
   bool ResolvedUndefs = true;
   while (ResolvedUndefs) {
     Solver.solve();
     LLVM_DEBUG(dbgs() << "RESOLVING UNDEFs\n");
     ResolvedUndefs = Solver.resolvedUndefsIn(F);
   }
 
   bool MadeChanges = false;
 
   // If we decided that there are basic blocks that are dead in this function,
   // delete their contents now.  Note that we cannot actually delete the blocks,
   // as we cannot modify the CFG of the function.
 
   SmallPtrSet<Value *, 32> InsertedValues;
   for (BasicBlock &BB : F) {
     if (!Solver.isBlockExecutable(&BB)) {
       LLVM_DEBUG(dbgs() << "  BasicBlock Dead:" << BB);
 
       ++NumDeadBlocks;
       NumInstRemoved += removeAllNonTerminatorAndEHPadInstructions(&BB).first;
 
       MadeChanges = true;
       continue;
     }
 
     MadeChanges |= simplifyInstsInBlock(Solver, BB, InsertedValues,
                                         NumInstRemoved, NumInstReplaced);
   }
 
   return MadeChanges;
 }
 
 PreservedAnalyses SCCPPass::run(Function &F, FunctionAnalysisManager &AM) {
   const DataLayout &DL = F.getParent()->getDataLayout();
   auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
   if (!runSCCP(F, DL, &TLI))
     return PreservedAnalyses::all();
 
   auto PA = PreservedAnalyses();
   PA.preserveSet<CFGAnalyses>();
   return PA;
 }
 
 namespace {
 
 //===--------------------------------------------------------------------===//
 //
 /// SCCP Class - This class uses the SCCPSolver to implement a per-function
 /// Sparse Conditional Constant Propagator.
 ///
 class SCCPLegacyPass : public FunctionPass {
 public:
   // Pass identification, replacement for typeid
   static char ID;
 
   SCCPLegacyPass() : FunctionPass(ID) {
     initializeSCCPLegacyPassPass(*PassRegistry::getPassRegistry());
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<TargetLibraryInfoWrapperPass>();
     AU.addPreserved<GlobalsAAWrapperPass>();
     AU.setPreservesCFG();
   }
 
   // runOnFunction - Run the Sparse Conditional Constant Propagation
   // algorithm, and return true if the function was modified.
   bool runOnFunction(Function &F) override {
     if (skipFunction(F))
       return false;
     const DataLayout &DL = F.getParent()->getDataLayout();
     const TargetLibraryInfo *TLI =
         &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
     return runSCCP(F, DL, TLI);
   }
 };
 
 } // end anonymous namespace
 
 char SCCPLegacyPass::ID = 0;
 
 INITIALIZE_PASS_BEGIN(SCCPLegacyPass, "sccp",
                       "Sparse Conditional Constant Propagation", false, false)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 INITIALIZE_PASS_END(SCCPLegacyPass, "sccp",
                     "Sparse Conditional Constant Propagation", false, false)
 
 // createSCCPPass - This is the public interface to this file.
 FunctionPass *llvm::createSCCPPass() { return new SCCPLegacyPass(); }
 
 static void findReturnsToZap(Function &F,
                              SmallVector<ReturnInst *, 8> &ReturnsToZap,
                              SCCPSolver &Solver) {
   // We can only do this if we know that nothing else can call the function.
   if (!Solver.isArgumentTrackedFunction(&F))
     return;
 
   if (Solver.mustPreserveReturn(&F)) {
     LLVM_DEBUG(
         dbgs()
         << "Can't zap returns of the function : " << F.getName()
         << " due to present musttail or \"clang.arc.attachedcall\" call of "
            "it\n");
     return;
   }
 
   assert(
       all_of(F.users(),
              [&Solver](User *U) {
                if (isa<Instruction>(U) &&
                    !Solver.isBlockExecutable(cast<Instruction>(U)->getParent()))
                  return true;
                // Non-callsite uses are not impacted by zapping. Also, constant
                // uses (like blockaddresses) could stuck around, without being
                // used in the underlying IR, meaning we do not have lattice
                // values for them.
                if (!isa<CallBase>(U))
                  return true;
                if (U->getType()->isStructTy()) {
                  return all_of(Solver.getStructLatticeValueFor(U),
                                [](const ValueLatticeElement &LV) {
                                  return !isOverdefined(LV);
                                });
                }
                return !isOverdefined(Solver.getLatticeValueFor(U));
              }) &&
       "We can only zap functions where all live users have a concrete value");
 
   for (BasicBlock &BB : F) {
     if (CallInst *CI = BB.getTerminatingMustTailCall()) {
       LLVM_DEBUG(dbgs() << "Can't zap return of the block due to present "
                         << "musttail call : " << *CI << "\n");
       (void)CI;
       return;
     }
 
     if (auto *RI = dyn_cast<ReturnInst>(BB.getTerminator()))
       if (!isa<UndefValue>(RI->getOperand(0)))
         ReturnsToZap.push_back(RI);
   }
 }
 
 static bool removeNonFeasibleEdges(const SCCPSolver &Solver, BasicBlock *BB,
-                                   DomTreeUpdater &DTU) {
+                                   DomTreeUpdater &DTU,
+                                   BasicBlock *&NewUnreachableBB) {
   SmallPtrSet<BasicBlock *, 8> FeasibleSuccessors;
   bool HasNonFeasibleEdges = false;
   for (BasicBlock *Succ : successors(BB)) {
     if (Solver.isEdgeFeasible(BB, Succ))
       FeasibleSuccessors.insert(Succ);
     else
       HasNonFeasibleEdges = true;
   }
 
   // All edges feasible, nothing to do.
   if (!HasNonFeasibleEdges)
     return false;
 
   // SCCP can only determine non-feasible edges for br, switch and indirectbr.
   Instruction *TI = BB->getTerminator();
   assert((isa<BranchInst>(TI) || isa<SwitchInst>(TI) ||
           isa<IndirectBrInst>(TI)) &&
          "Terminator must be a br, switch or indirectbr");
 
   if (FeasibleSuccessors.size() == 1) {
     // Replace with an unconditional branch to the only feasible successor.
     BasicBlock *OnlyFeasibleSuccessor = *FeasibleSuccessors.begin();
     SmallVector<DominatorTree::UpdateType, 8> Updates;
     bool HaveSeenOnlyFeasibleSuccessor = false;
     for (BasicBlock *Succ : successors(BB)) {
       if (Succ == OnlyFeasibleSuccessor && !HaveSeenOnlyFeasibleSuccessor) {
         // Don't remove the edge to the only feasible successor the first time
         // we see it. We still do need to remove any multi-edges to it though.
         HaveSeenOnlyFeasibleSuccessor = true;
         continue;
       }
 
       Succ->removePredecessor(BB);
       Updates.push_back({DominatorTree::Delete, BB, Succ});
     }
 
     BranchInst::Create(OnlyFeasibleSuccessor, BB);
     TI->eraseFromParent();
     DTU.applyUpdatesPermissive(Updates);
   } else if (FeasibleSuccessors.size() > 1) {
     SwitchInstProfUpdateWrapper SI(*cast<SwitchInst>(TI));
     SmallVector<DominatorTree::UpdateType, 8> Updates;
+
+    // If the default destination is unfeasible it will never be taken. Replace
+    // it with a new block with a single Unreachable instruction.
+    BasicBlock *DefaultDest = SI->getDefaultDest();
+    if (!FeasibleSuccessors.contains(DefaultDest)) {
+      if (!NewUnreachableBB) {
+        NewUnreachableBB =
+            BasicBlock::Create(DefaultDest->getContext(), "default.unreachable",
+                               DefaultDest->getParent(), DefaultDest);
+        new UnreachableInst(DefaultDest->getContext(), NewUnreachableBB);
+      }
+
+      SI->setDefaultDest(NewUnreachableBB);
+      Updates.push_back({DominatorTree::Delete, BB, DefaultDest});
+      Updates.push_back({DominatorTree::Insert, BB, NewUnreachableBB});
+    }
+
     for (auto CI = SI->case_begin(); CI != SI->case_end();) {
       if (FeasibleSuccessors.contains(CI->getCaseSuccessor())) {
         ++CI;
         continue;
       }
 
       BasicBlock *Succ = CI->getCaseSuccessor();
       Succ->removePredecessor(BB);
       Updates.push_back({DominatorTree::Delete, BB, Succ});
       SI.removeCase(CI);
       // Don't increment CI, as we removed a case.
     }
 
     DTU.applyUpdatesPermissive(Updates);
   } else {
     llvm_unreachable("Must have at least one feasible successor");
   }
   return true;
 }
 
 bool llvm::runIPSCCP(
     Module &M, const DataLayout &DL,
     std::function<const TargetLibraryInfo &(Function &)> GetTLI,
     function_ref<AnalysisResultsForFn(Function &)> getAnalysis) {
   SCCPSolver Solver(DL, GetTLI, M.getContext());
 
   // Loop over all functions, marking arguments to those with their addresses
   // taken or that are external as overdefined.
   for (Function &F : M) {
     if (F.isDeclaration())
       continue;
 
     Solver.addAnalysis(F, getAnalysis(F));
 
     // Determine if we can track the function's return values. If so, add the
     // function to the solver's set of return-tracked functions.
     if (canTrackReturnsInterprocedurally(&F))
       Solver.addTrackedFunction(&F);
 
     // Determine if we can track the function's arguments. If so, add the
     // function to the solver's set of argument-tracked functions.
     if (canTrackArgumentsInterprocedurally(&F)) {
       Solver.addArgumentTrackedFunction(&F);
       continue;
     }
 
     // Assume the function is called.
     Solver.markBlockExecutable(&F.front());
 
     // Assume nothing about the incoming arguments.
     for (Argument &AI : F.args())
       Solver.markOverdefined(&AI);
   }
 
   // Determine if we can track any of the module's global variables. If so, add
   // the global variables we can track to the solver's set of tracked global
   // variables.
   for (GlobalVariable &G : M.globals()) {
     G.removeDeadConstantUsers();
     if (canTrackGlobalVariableInterprocedurally(&G))
       Solver.trackValueOfGlobalVariable(&G);
   }
 
   // Solve for constants.
   bool ResolvedUndefs = true;
   Solver.solve();
   while (ResolvedUndefs) {
     LLVM_DEBUG(dbgs() << "RESOLVING UNDEFS\n");
     ResolvedUndefs = false;
     for (Function &F : M) {
       if (Solver.resolvedUndefsIn(F))
         ResolvedUndefs = true;
     }
     if (ResolvedUndefs)
       Solver.solve();
   }
 
   bool MadeChanges = false;
 
   // Iterate over all of the instructions in the module, replacing them with
   // constants if we have found them to be of constant values.
 
   for (Function &F : M) {
     if (F.isDeclaration())
       continue;
 
     SmallVector<BasicBlock *, 512> BlocksToErase;
 
     if (Solver.isBlockExecutable(&F.front())) {
       bool ReplacedPointerArg = false;
       for (Argument &Arg : F.args()) {
         if (!Arg.use_empty() && tryToReplaceWithConstant(Solver, &Arg)) {
           ReplacedPointerArg |= Arg.getType()->isPointerTy();
           ++IPNumArgsElimed;
         }
       }
 
       // If we replaced an argument, the argmemonly and
       // inaccessiblemem_or_argmemonly attributes do not hold any longer. Remove
       // them from both the function and callsites.
       if (ReplacedPointerArg) {
         AttributeMask AttributesToRemove;
         AttributesToRemove.addAttribute(Attribute::ArgMemOnly);
         AttributesToRemove.addAttribute(Attribute::InaccessibleMemOrArgMemOnly);
         F.removeFnAttrs(AttributesToRemove);
 
         for (User *U : F.users()) {
           auto *CB = dyn_cast<CallBase>(U);
           if (!CB || CB->getCalledFunction() != &F)
             continue;
 
           CB->removeFnAttrs(AttributesToRemove);
         }
       }
       MadeChanges |= ReplacedPointerArg;
     }
 
     SmallPtrSet<Value *, 32> InsertedValues;
     for (BasicBlock &BB : F) {
       if (!Solver.isBlockExecutable(&BB)) {
         LLVM_DEBUG(dbgs() << "  BasicBlock Dead:" << BB);
         ++NumDeadBlocks;
 
         MadeChanges = true;
 
         if (&BB != &F.front())
           BlocksToErase.push_back(&BB);
         continue;
       }
 
       MadeChanges |= simplifyInstsInBlock(Solver, BB, InsertedValues,
                                           IPNumInstRemoved, IPNumInstReplaced);
     }
 
     DomTreeUpdater DTU = Solver.getDTU(F);
     // Change dead blocks to unreachable. We do it after replacing constants
     // in all executable blocks, because changeToUnreachable may remove PHI
     // nodes in executable blocks we found values for. The function's entry
     // block is not part of BlocksToErase, so we have to handle it separately.
     for (BasicBlock *BB : BlocksToErase) {
       NumInstRemoved += changeToUnreachable(BB->getFirstNonPHI(),
                                             /*PreserveLCSSA=*/false, &DTU);
     }
     if (!Solver.isBlockExecutable(&F.front()))
       NumInstRemoved += changeToUnreachable(F.front().getFirstNonPHI(),
                                             /*PreserveLCSSA=*/false, &DTU);
 
+    BasicBlock *NewUnreachableBB = nullptr;
     for (BasicBlock &BB : F)
-      MadeChanges |= removeNonFeasibleEdges(Solver, &BB, DTU);
+      MadeChanges |= removeNonFeasibleEdges(Solver, &BB, DTU, NewUnreachableBB);
 
     for (BasicBlock *DeadBB : BlocksToErase)
       DTU.deleteBB(DeadBB);
 
     for (BasicBlock &BB : F) {
       for (Instruction &Inst : llvm::make_early_inc_range(BB)) {
         if (Solver.getPredicateInfoFor(&Inst)) {
           if (auto *II = dyn_cast<IntrinsicInst>(&Inst)) {
             if (II->getIntrinsicID() == Intrinsic::ssa_copy) {
               Value *Op = II->getOperand(0);
               Inst.replaceAllUsesWith(Op);
               Inst.eraseFromParent();
             }
           }
         }
       }
     }
   }
 
   // If we inferred constant or undef return values for a function, we replaced
   // all call uses with the inferred value.  This means we don't need to bother
   // actually returning anything from the function.  Replace all return
   // instructions with return undef.
   //
   // Do this in two stages: first identify the functions we should process, then
   // actually zap their returns.  This is important because we can only do this
   // if the address of the function isn't taken.  In cases where a return is the
   // last use of a function, the order of processing functions would affect
   // whether other functions are optimizable.
   SmallVector<ReturnInst*, 8> ReturnsToZap;
 
   for (const auto &I : Solver.getTrackedRetVals()) {
     Function *F = I.first;
     const ValueLatticeElement &ReturnValue = I.second;
 
     // If there is a known constant range for the return value, add !range
     // metadata to the function's call sites.
     if (ReturnValue.isConstantRange() &&
         !ReturnValue.getConstantRange().isSingleElement()) {
       // Do not add range metadata if the return value may include undef.
       if (ReturnValue.isConstantRangeIncludingUndef())
         continue;
 
       auto &CR = ReturnValue.getConstantRange();
       for (User *User : F->users()) {
         auto *CB = dyn_cast<CallBase>(User);
         if (!CB || CB->getCalledFunction() != F)
           continue;
 
         // Limit to cases where the return value is guaranteed to be neither
         // poison nor undef. Poison will be outside any range and currently
         // values outside of the specified range cause immediate undefined
         // behavior.
         if (!isGuaranteedNotToBeUndefOrPoison(CB, nullptr, CB))
           continue;
 
         // Do not touch existing metadata for now.
         // TODO: We should be able to take the intersection of the existing
         // metadata and the inferred range.
         if (CB->getMetadata(LLVMContext::MD_range))
           continue;
 
         LLVMContext &Context = CB->getParent()->getContext();
         Metadata *RangeMD[] = {
             ConstantAsMetadata::get(ConstantInt::get(Context, CR.getLower())),
             ConstantAsMetadata::get(ConstantInt::get(Context, CR.getUpper()))};
         CB->setMetadata(LLVMContext::MD_range, MDNode::get(Context, RangeMD));
       }
       continue;
     }
     if (F->getReturnType()->isVoidTy())
       continue;
     if (isConstant(ReturnValue) || ReturnValue.isUnknownOrUndef())
       findReturnsToZap(*F, ReturnsToZap, Solver);
   }
 
   for (auto F : Solver.getMRVFunctionsTracked()) {
     assert(F->getReturnType()->isStructTy() &&
            "The return type should be a struct");
     StructType *STy = cast<StructType>(F->getReturnType());
     if (Solver.isStructLatticeConstant(F, STy))
       findReturnsToZap(*F, ReturnsToZap, Solver);
   }
 
   // Zap all returns which we've identified as zap to change.
   SmallSetVector<Function *, 8> FuncZappedReturn;
   for (unsigned i = 0, e = ReturnsToZap.size(); i != e; ++i) {
     Function *F = ReturnsToZap[i]->getParent()->getParent();
     ReturnsToZap[i]->setOperand(0, UndefValue::get(F->getReturnType()));
     // Record all functions that are zapped.
     FuncZappedReturn.insert(F);
   }
 
   // Remove the returned attribute for zapped functions and the
   // corresponding call sites.
   for (Function *F : FuncZappedReturn) {
     for (Argument &A : F->args())
       F->removeParamAttr(A.getArgNo(), Attribute::Returned);
     for (Use &U : F->uses()) {
       // Skip over blockaddr users.
       if (isa<BlockAddress>(U.getUser()))
         continue;
       CallBase *CB = cast<CallBase>(U.getUser());
       for (Use &Arg : CB->args())
         CB->removeParamAttr(CB->getArgOperandNo(&Arg), Attribute::Returned);
     }
   }
 
   // If we inferred constant or undef values for globals variables, we can
   // delete the global and any stores that remain to it.
   for (auto &I : make_early_inc_range(Solver.getTrackedGlobals())) {
     GlobalVariable *GV = I.first;
     if (isOverdefined(I.second))
       continue;
     LLVM_DEBUG(dbgs() << "Found that GV '" << GV->getName()
                       << "' is constant!\n");
     while (!GV->use_empty()) {
       StoreInst *SI = cast<StoreInst>(GV->user_back());
       SI->eraseFromParent();
       MadeChanges = true;
     }
     M.getGlobalList().erase(GV);
     ++IPNumGlobalConst;
   }
 
   return MadeChanges;
 }