Index: projects/clang400-import/contrib/compiler-rt/lib/asan/asan_activation.cc =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/asan/asan_activation.cc (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/asan/asan_activation.cc (revision 312198) @@ -1,142 +1,144 @@ //===-- asan_activation.cc --------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file is a part of AddressSanitizer, an address sanity checker. // // ASan activation/deactivation logic. //===----------------------------------------------------------------------===// #include "asan_activation.h" #include "asan_allocator.h" #include "asan_flags.h" #include "asan_internal.h" #include "asan_poisoning.h" #include "asan_stack.h" #include "sanitizer_common/sanitizer_flags.h" namespace __asan { static struct AsanDeactivatedFlags { AllocatorOptions allocator_options; int malloc_context_size; bool poison_heap; bool coverage; const char *coverage_dir; void RegisterActivationFlags(FlagParser *parser, Flags *f, CommonFlags *cf) { #define ASAN_ACTIVATION_FLAG(Type, Name) \ RegisterFlag(parser, #Name, "", &f->Name); #define COMMON_ACTIVATION_FLAG(Type, Name) \ RegisterFlag(parser, #Name, "", &cf->Name); #include "asan_activation_flags.inc" #undef ASAN_ACTIVATION_FLAG #undef COMMON_ACTIVATION_FLAG RegisterIncludeFlags(parser, cf); } void OverrideFromActivationFlags() { Flags f; CommonFlags cf; FlagParser parser; RegisterActivationFlags(&parser, &f, &cf); cf.SetDefaults(); // Copy the current activation flags. allocator_options.CopyTo(&f, &cf); cf.malloc_context_size = malloc_context_size; f.poison_heap = poison_heap; cf.coverage = coverage; cf.coverage_dir = coverage_dir; cf.verbosity = Verbosity(); cf.help = false; // this is activation-specific help // Check if activation flags need to be overriden. if (const char *env = GetEnv("ASAN_ACTIVATION_OPTIONS")) { parser.ParseString(env); } InitializeCommonFlags(&cf); if (Verbosity()) ReportUnrecognizedFlags(); if (cf.help) parser.PrintFlagDescriptions(); allocator_options.SetFrom(&f, &cf); malloc_context_size = cf.malloc_context_size; poison_heap = f.poison_heap; coverage = cf.coverage; coverage_dir = cf.coverage_dir; } void Print() { Report( - "quarantine_size_mb %d, max_redzone %d, poison_heap %d, " - "malloc_context_size %d, alloc_dealloc_mismatch %d, " - "allocator_may_return_null %d, coverage %d, coverage_dir %s, " - "allocator_release_to_os_interval_ms %d\n", - allocator_options.quarantine_size_mb, allocator_options.max_redzone, - poison_heap, malloc_context_size, + "quarantine_size_mb %d, thread_local_quarantine_size_kb %d, " + "max_redzone %d, poison_heap %d, malloc_context_size %d, " + "alloc_dealloc_mismatch %d, allocator_may_return_null %d, coverage %d, " + "coverage_dir %s, allocator_release_to_os_interval_ms %d\n", + allocator_options.quarantine_size_mb, + allocator_options.thread_local_quarantine_size_kb, + allocator_options.max_redzone, poison_heap, malloc_context_size, allocator_options.alloc_dealloc_mismatch, allocator_options.may_return_null, coverage, coverage_dir, allocator_options.release_to_os_interval_ms); } } asan_deactivated_flags; static bool asan_is_deactivated; void AsanDeactivate() { CHECK(!asan_is_deactivated); VReport(1, "Deactivating ASan\n"); // Stash runtime state. GetAllocatorOptions(&asan_deactivated_flags.allocator_options); asan_deactivated_flags.malloc_context_size = GetMallocContextSize(); asan_deactivated_flags.poison_heap = CanPoisonMemory(); asan_deactivated_flags.coverage = common_flags()->coverage; asan_deactivated_flags.coverage_dir = common_flags()->coverage_dir; // Deactivate the runtime. SetCanPoisonMemory(false); SetMallocContextSize(1); ReInitializeCoverage(false, nullptr); AllocatorOptions disabled = asan_deactivated_flags.allocator_options; disabled.quarantine_size_mb = 0; + disabled.thread_local_quarantine_size_kb = 0; disabled.min_redzone = 16; // Redzone must be at least 16 bytes long. disabled.max_redzone = 16; disabled.alloc_dealloc_mismatch = false; disabled.may_return_null = true; ReInitializeAllocator(disabled); asan_is_deactivated = true; } void AsanActivate() { if (!asan_is_deactivated) return; VReport(1, "Activating ASan\n"); UpdateProcessName(); asan_deactivated_flags.OverrideFromActivationFlags(); SetCanPoisonMemory(asan_deactivated_flags.poison_heap); SetMallocContextSize(asan_deactivated_flags.malloc_context_size); ReInitializeCoverage(asan_deactivated_flags.coverage, asan_deactivated_flags.coverage_dir); ReInitializeAllocator(asan_deactivated_flags.allocator_options); asan_is_deactivated = false; if (Verbosity()) { Report("Activated with flags:\n"); asan_deactivated_flags.Print(); } } } // namespace __asan Index: projects/clang400-import/contrib/compiler-rt/lib/asan/asan_activation_flags.inc =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/asan/asan_activation_flags.inc (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/asan/asan_activation_flags.inc (revision 312198) @@ -1,36 +1,37 @@ //===-- asan_activation_flags.inc -------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // A subset of ASan (and common) runtime flags supported at activation time. // //===----------------------------------------------------------------------===// #ifndef ASAN_ACTIVATION_FLAG # error "Define ASAN_ACTIVATION_FLAG prior to including this file!" #endif #ifndef COMMON_ACTIVATION_FLAG # error "Define COMMON_ACTIVATION_FLAG prior to including this file!" #endif // ASAN_ACTIVATION_FLAG(Type, Name) // See COMMON_FLAG in sanitizer_flags.inc for more details. ASAN_ACTIVATION_FLAG(int, redzone) ASAN_ACTIVATION_FLAG(int, max_redzone) ASAN_ACTIVATION_FLAG(int, quarantine_size_mb) +ASAN_ACTIVATION_FLAG(int, thread_local_quarantine_size_kb) ASAN_ACTIVATION_FLAG(bool, alloc_dealloc_mismatch) ASAN_ACTIVATION_FLAG(bool, poison_heap) COMMON_ACTIVATION_FLAG(bool, allocator_may_return_null) COMMON_ACTIVATION_FLAG(int, malloc_context_size) COMMON_ACTIVATION_FLAG(bool, coverage) COMMON_ACTIVATION_FLAG(const char *, coverage_dir) COMMON_ACTIVATION_FLAG(int, verbosity) COMMON_ACTIVATION_FLAG(bool, help) COMMON_ACTIVATION_FLAG(s32, allocator_release_to_os_interval_ms) Index: projects/clang400-import/contrib/compiler-rt/lib/asan/asan_allocator.cc =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/asan/asan_allocator.cc (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/asan/asan_allocator.cc (revision 312198) @@ -1,972 +1,972 @@ //===-- asan_allocator.cc -------------------------------------------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file is a part of AddressSanitizer, an address sanity checker. // // Implementation of ASan's memory allocator, 2-nd version. // This variant uses the allocator from sanitizer_common, i.e. the one shared // with ThreadSanitizer and MemorySanitizer. // //===----------------------------------------------------------------------===// #include "asan_allocator.h" #include "asan_mapping.h" #include "asan_poisoning.h" #include "asan_report.h" #include "asan_stack.h" #include "asan_thread.h" #include "sanitizer_common/sanitizer_allocator_interface.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_list.h" #include "sanitizer_common/sanitizer_stackdepot.h" #include "sanitizer_common/sanitizer_quarantine.h" #include "lsan/lsan_common.h" namespace __asan { // Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits. // We use adaptive redzones: for larger allocation larger redzones are used. static u32 RZLog2Size(u32 rz_log) { CHECK_LT(rz_log, 8); return 16 << rz_log; } static u32 RZSize2Log(u32 rz_size) { CHECK_GE(rz_size, 16); CHECK_LE(rz_size, 2048); CHECK(IsPowerOfTwo(rz_size)); u32 res = Log2(rz_size) - 4; CHECK_EQ(rz_size, RZLog2Size(res)); return res; } static AsanAllocator &get_allocator(); // The memory chunk allocated from the underlying allocator looks like this: // L L L L L L H H U U U U U U R R // L -- left redzone words (0 or more bytes) // H -- ChunkHeader (16 bytes), which is also a part of the left redzone. // U -- user memory. // R -- right redzone (0 or more bytes) // ChunkBase consists of ChunkHeader and other bytes that overlap with user // memory. // If the left redzone is greater than the ChunkHeader size we store a magic // value in the first uptr word of the memory block and store the address of // ChunkBase in the next uptr. // M B L L L L L L L L L H H U U U U U U // | ^ // ---------------------| // M -- magic value kAllocBegMagic // B -- address of ChunkHeader pointing to the first 'H' static const uptr kAllocBegMagic = 0xCC6E96B9; struct ChunkHeader { // 1-st 8 bytes. u32 chunk_state : 8; // Must be first. u32 alloc_tid : 24; u32 free_tid : 24; u32 from_memalign : 1; u32 alloc_type : 2; u32 rz_log : 3; u32 lsan_tag : 2; // 2-nd 8 bytes // This field is used for small sizes. For large sizes it is equal to // SizeClassMap::kMaxSize and the actual size is stored in the // SecondaryAllocator's metadata. u32 user_requested_size; u32 alloc_context_id; }; struct ChunkBase : ChunkHeader { // Header2, intersects with user memory. u32 free_context_id; }; static const uptr kChunkHeaderSize = sizeof(ChunkHeader); static const uptr kChunkHeader2Size = sizeof(ChunkBase) - kChunkHeaderSize; COMPILER_CHECK(kChunkHeaderSize == 16); COMPILER_CHECK(kChunkHeader2Size <= 16); // Every chunk of memory allocated by this allocator can be in one of 3 states: // CHUNK_AVAILABLE: the chunk is in the free list and ready to be allocated. // CHUNK_ALLOCATED: the chunk is allocated and not yet freed. // CHUNK_QUARANTINE: the chunk was freed and put into quarantine zone. enum { CHUNK_AVAILABLE = 0, // 0 is the default value even if we didn't set it. CHUNK_ALLOCATED = 2, CHUNK_QUARANTINE = 3 }; struct AsanChunk: ChunkBase { uptr Beg() { return reinterpret_cast(this) + kChunkHeaderSize; } uptr UsedSize(bool locked_version = false) { if (user_requested_size != SizeClassMap::kMaxSize) return user_requested_size; return *reinterpret_cast( get_allocator().GetMetaData(AllocBeg(locked_version))); } void *AllocBeg(bool locked_version = false) { if (from_memalign) { if (locked_version) return get_allocator().GetBlockBeginFastLocked( reinterpret_cast(this)); return get_allocator().GetBlockBegin(reinterpret_cast(this)); } return reinterpret_cast(Beg() - RZLog2Size(rz_log)); } bool AddrIsInside(uptr addr, bool locked_version = false) { return (addr >= Beg()) && (addr < Beg() + UsedSize(locked_version)); } }; struct QuarantineCallback { explicit QuarantineCallback(AllocatorCache *cache) : cache_(cache) { } void Recycle(AsanChunk *m) { CHECK_EQ(m->chunk_state, CHUNK_QUARANTINE); atomic_store((atomic_uint8_t*)m, CHUNK_AVAILABLE, memory_order_relaxed); CHECK_NE(m->alloc_tid, kInvalidTid); CHECK_NE(m->free_tid, kInvalidTid); PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY), kAsanHeapLeftRedzoneMagic); void *p = reinterpret_cast(m->AllocBeg()); if (p != m) { uptr *alloc_magic = reinterpret_cast(p); CHECK_EQ(alloc_magic[0], kAllocBegMagic); // Clear the magic value, as allocator internals may overwrite the // contents of deallocated chunk, confusing GetAsanChunk lookup. alloc_magic[0] = 0; CHECK_EQ(alloc_magic[1], reinterpret_cast(m)); } // Statistics. AsanStats &thread_stats = GetCurrentThreadStats(); thread_stats.real_frees++; thread_stats.really_freed += m->UsedSize(); get_allocator().Deallocate(cache_, p); } void *Allocate(uptr size) { return get_allocator().Allocate(cache_, size, 1, false); } void Deallocate(void *p) { get_allocator().Deallocate(cache_, p); } AllocatorCache *cache_; }; typedef Quarantine AsanQuarantine; typedef AsanQuarantine::Cache QuarantineCache; void AsanMapUnmapCallback::OnMap(uptr p, uptr size) const { PoisonShadow(p, size, kAsanHeapLeftRedzoneMagic); // Statistics. AsanStats &thread_stats = GetCurrentThreadStats(); thread_stats.mmaps++; thread_stats.mmaped += size; } void AsanMapUnmapCallback::OnUnmap(uptr p, uptr size) const { PoisonShadow(p, size, 0); // We are about to unmap a chunk of user memory. // Mark the corresponding shadow memory as not needed. FlushUnneededASanShadowMemory(p, size); // Statistics. AsanStats &thread_stats = GetCurrentThreadStats(); thread_stats.munmaps++; thread_stats.munmaped += size; } // We can not use THREADLOCAL because it is not supported on some of the // platforms we care about (OSX 10.6, Android). // static THREADLOCAL AllocatorCache cache; AllocatorCache *GetAllocatorCache(AsanThreadLocalMallocStorage *ms) { CHECK(ms); return &ms->allocator_cache; } QuarantineCache *GetQuarantineCache(AsanThreadLocalMallocStorage *ms) { CHECK(ms); CHECK_LE(sizeof(QuarantineCache), sizeof(ms->quarantine_cache)); return reinterpret_cast(ms->quarantine_cache); } void AllocatorOptions::SetFrom(const Flags *f, const CommonFlags *cf) { quarantine_size_mb = f->quarantine_size_mb; thread_local_quarantine_size_kb = f->thread_local_quarantine_size_kb; min_redzone = f->redzone; max_redzone = f->max_redzone; may_return_null = cf->allocator_may_return_null; alloc_dealloc_mismatch = f->alloc_dealloc_mismatch; release_to_os_interval_ms = cf->allocator_release_to_os_interval_ms; } void AllocatorOptions::CopyTo(Flags *f, CommonFlags *cf) { f->quarantine_size_mb = quarantine_size_mb; f->thread_local_quarantine_size_kb = thread_local_quarantine_size_kb; f->redzone = min_redzone; f->max_redzone = max_redzone; cf->allocator_may_return_null = may_return_null; f->alloc_dealloc_mismatch = alloc_dealloc_mismatch; cf->allocator_release_to_os_interval_ms = release_to_os_interval_ms; } struct Allocator { static const uptr kMaxAllowedMallocSize = FIRST_32_SECOND_64(3UL << 30, 1ULL << 40); AsanAllocator allocator; AsanQuarantine quarantine; StaticSpinMutex fallback_mutex; AllocatorCache fallback_allocator_cache; QuarantineCache fallback_quarantine_cache; // ------------------- Options -------------------------- atomic_uint16_t min_redzone; atomic_uint16_t max_redzone; atomic_uint8_t alloc_dealloc_mismatch; // ------------------- Initialization ------------------------ explicit Allocator(LinkerInitialized) : quarantine(LINKER_INITIALIZED), fallback_quarantine_cache(LINKER_INITIALIZED) {} void CheckOptions(const AllocatorOptions &options) const { CHECK_GE(options.min_redzone, 16); CHECK_GE(options.max_redzone, options.min_redzone); CHECK_LE(options.max_redzone, 2048); CHECK(IsPowerOfTwo(options.min_redzone)); CHECK(IsPowerOfTwo(options.max_redzone)); } void SharedInitCode(const AllocatorOptions &options) { CheckOptions(options); quarantine.Init((uptr)options.quarantine_size_mb << 20, (uptr)options.thread_local_quarantine_size_kb << 10); atomic_store(&alloc_dealloc_mismatch, options.alloc_dealloc_mismatch, memory_order_release); atomic_store(&min_redzone, options.min_redzone, memory_order_release); atomic_store(&max_redzone, options.max_redzone, memory_order_release); } void Initialize(const AllocatorOptions &options) { allocator.Init(options.may_return_null, options.release_to_os_interval_ms); SharedInitCode(options); } void RePoisonChunk(uptr chunk) { - // This could a user-facing chunk (with redzones), or some internal + // This could be a user-facing chunk (with redzones), or some internal // housekeeping chunk, like TransferBatch. Start by assuming the former. AsanChunk *ac = GetAsanChunk((void *)chunk); uptr allocated_size = allocator.GetActuallyAllocatedSize((void *)ac); uptr beg = ac->Beg(); uptr end = ac->Beg() + ac->UsedSize(true); uptr chunk_end = chunk + allocated_size; - if (chunk < beg && beg < end && end <= chunk_end) { - // Looks like a valid AsanChunk. Or maybe not. Be conservative and only - // poison the redzones. + if (chunk < beg && beg < end && end <= chunk_end && + ac->chunk_state == CHUNK_ALLOCATED) { + // Looks like a valid AsanChunk in use, poison redzones only. PoisonShadow(chunk, beg - chunk, kAsanHeapLeftRedzoneMagic); uptr end_aligned_down = RoundDownTo(end, SHADOW_GRANULARITY); FastPoisonShadowPartialRightRedzone( end_aligned_down, end - end_aligned_down, chunk_end - end_aligned_down, kAsanHeapLeftRedzoneMagic); } else { - // This can not be an AsanChunk. Poison everything. It may be reused as - // AsanChunk later. + // This is either not an AsanChunk or freed or quarantined AsanChunk. + // In either case, poison everything. PoisonShadow(chunk, allocated_size, kAsanHeapLeftRedzoneMagic); } } void ReInitialize(const AllocatorOptions &options) { allocator.SetMayReturnNull(options.may_return_null); allocator.SetReleaseToOSIntervalMs(options.release_to_os_interval_ms); SharedInitCode(options); // Poison all existing allocation's redzones. if (CanPoisonMemory()) { allocator.ForceLock(); allocator.ForEachChunk( [](uptr chunk, void *alloc) { ((Allocator *)alloc)->RePoisonChunk(chunk); }, this); allocator.ForceUnlock(); } } void GetOptions(AllocatorOptions *options) const { options->quarantine_size_mb = quarantine.GetSize() >> 20; options->thread_local_quarantine_size_kb = quarantine.GetCacheSize() >> 10; options->min_redzone = atomic_load(&min_redzone, memory_order_acquire); options->max_redzone = atomic_load(&max_redzone, memory_order_acquire); options->may_return_null = allocator.MayReturnNull(); options->alloc_dealloc_mismatch = atomic_load(&alloc_dealloc_mismatch, memory_order_acquire); options->release_to_os_interval_ms = allocator.ReleaseToOSIntervalMs(); } // -------------------- Helper methods. ------------------------- uptr ComputeRZLog(uptr user_requested_size) { u32 rz_log = user_requested_size <= 64 - 16 ? 0 : user_requested_size <= 128 - 32 ? 1 : user_requested_size <= 512 - 64 ? 2 : user_requested_size <= 4096 - 128 ? 3 : user_requested_size <= (1 << 14) - 256 ? 4 : user_requested_size <= (1 << 15) - 512 ? 5 : user_requested_size <= (1 << 16) - 1024 ? 6 : 7; u32 min_rz = atomic_load(&min_redzone, memory_order_acquire); u32 max_rz = atomic_load(&max_redzone, memory_order_acquire); return Min(Max(rz_log, RZSize2Log(min_rz)), RZSize2Log(max_rz)); } // We have an address between two chunks, and we want to report just one. AsanChunk *ChooseChunk(uptr addr, AsanChunk *left_chunk, AsanChunk *right_chunk) { // Prefer an allocated chunk over freed chunk and freed chunk // over available chunk. if (left_chunk->chunk_state != right_chunk->chunk_state) { if (left_chunk->chunk_state == CHUNK_ALLOCATED) return left_chunk; if (right_chunk->chunk_state == CHUNK_ALLOCATED) return right_chunk; if (left_chunk->chunk_state == CHUNK_QUARANTINE) return left_chunk; if (right_chunk->chunk_state == CHUNK_QUARANTINE) return right_chunk; } // Same chunk_state: choose based on offset. sptr l_offset = 0, r_offset = 0; CHECK(AsanChunkView(left_chunk).AddrIsAtRight(addr, 1, &l_offset)); CHECK(AsanChunkView(right_chunk).AddrIsAtLeft(addr, 1, &r_offset)); if (l_offset < r_offset) return left_chunk; return right_chunk; } // -------------------- Allocation/Deallocation routines --------------- void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack, AllocType alloc_type, bool can_fill) { if (UNLIKELY(!asan_inited)) AsanInitFromRtl(); Flags &fl = *flags(); CHECK(stack); const uptr min_alignment = SHADOW_GRANULARITY; if (alignment < min_alignment) alignment = min_alignment; if (size == 0) { // We'd be happy to avoid allocating memory for zero-size requests, but // some programs/tests depend on this behavior and assume that malloc // would not return NULL even for zero-size allocations. Moreover, it // looks like operator new should never return NULL, and results of // consecutive "new" calls must be different even if the allocated size // is zero. size = 1; } CHECK(IsPowerOfTwo(alignment)); uptr rz_log = ComputeRZLog(size); uptr rz_size = RZLog2Size(rz_log); uptr rounded_size = RoundUpTo(Max(size, kChunkHeader2Size), alignment); uptr needed_size = rounded_size + rz_size; if (alignment > min_alignment) needed_size += alignment; bool using_primary_allocator = true; // If we are allocating from the secondary allocator, there will be no // automatic right redzone, so add the right redzone manually. if (!PrimaryAllocator::CanAllocate(needed_size, alignment)) { needed_size += rz_size; using_primary_allocator = false; } CHECK(IsAligned(needed_size, min_alignment)); if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize) { Report("WARNING: AddressSanitizer failed to allocate 0x%zx bytes\n", (void*)size); return allocator.ReturnNullOrDieOnBadRequest(); } AsanThread *t = GetCurrentThread(); void *allocated; bool check_rss_limit = true; if (t) { AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage()); allocated = allocator.Allocate(cache, needed_size, 8, false, check_rss_limit); } else { SpinMutexLock l(&fallback_mutex); AllocatorCache *cache = &fallback_allocator_cache; allocated = allocator.Allocate(cache, needed_size, 8, false, check_rss_limit); } if (!allocated) return allocator.ReturnNullOrDieOnOOM(); if (*(u8 *)MEM_TO_SHADOW((uptr)allocated) == 0 && CanPoisonMemory()) { // Heap poisoning is enabled, but the allocator provides an unpoisoned // chunk. This is possible if CanPoisonMemory() was false for some // time, for example, due to flags()->start_disabled. // Anyway, poison the block before using it for anything else. uptr allocated_size = allocator.GetActuallyAllocatedSize(allocated); PoisonShadow((uptr)allocated, allocated_size, kAsanHeapLeftRedzoneMagic); } uptr alloc_beg = reinterpret_cast(allocated); uptr alloc_end = alloc_beg + needed_size; uptr beg_plus_redzone = alloc_beg + rz_size; uptr user_beg = beg_plus_redzone; if (!IsAligned(user_beg, alignment)) user_beg = RoundUpTo(user_beg, alignment); uptr user_end = user_beg + size; CHECK_LE(user_end, alloc_end); uptr chunk_beg = user_beg - kChunkHeaderSize; AsanChunk *m = reinterpret_cast(chunk_beg); m->alloc_type = alloc_type; m->rz_log = rz_log; u32 alloc_tid = t ? t->tid() : 0; m->alloc_tid = alloc_tid; CHECK_EQ(alloc_tid, m->alloc_tid); // Does alloc_tid fit into the bitfield? m->free_tid = kInvalidTid; m->from_memalign = user_beg != beg_plus_redzone; if (alloc_beg != chunk_beg) { CHECK_LE(alloc_beg+ 2 * sizeof(uptr), chunk_beg); reinterpret_cast(alloc_beg)[0] = kAllocBegMagic; reinterpret_cast(alloc_beg)[1] = chunk_beg; } if (using_primary_allocator) { CHECK(size); m->user_requested_size = size; CHECK(allocator.FromPrimary(allocated)); } else { CHECK(!allocator.FromPrimary(allocated)); m->user_requested_size = SizeClassMap::kMaxSize; uptr *meta = reinterpret_cast(allocator.GetMetaData(allocated)); meta[0] = size; meta[1] = chunk_beg; } m->alloc_context_id = StackDepotPut(*stack); uptr size_rounded_down_to_granularity = RoundDownTo(size, SHADOW_GRANULARITY); // Unpoison the bulk of the memory region. if (size_rounded_down_to_granularity) PoisonShadow(user_beg, size_rounded_down_to_granularity, 0); // Deal with the end of the region if size is not aligned to granularity. if (size != size_rounded_down_to_granularity && CanPoisonMemory()) { u8 *shadow = (u8 *)MemToShadow(user_beg + size_rounded_down_to_granularity); *shadow = fl.poison_partial ? (size & (SHADOW_GRANULARITY - 1)) : 0; } AsanStats &thread_stats = GetCurrentThreadStats(); thread_stats.mallocs++; thread_stats.malloced += size; thread_stats.malloced_redzones += needed_size - size; if (needed_size > SizeClassMap::kMaxSize) thread_stats.malloc_large++; else thread_stats.malloced_by_size[SizeClassMap::ClassID(needed_size)]++; void *res = reinterpret_cast(user_beg); if (can_fill && fl.max_malloc_fill_size) { uptr fill_size = Min(size, (uptr)fl.max_malloc_fill_size); REAL(memset)(res, fl.malloc_fill_byte, fill_size); } #if CAN_SANITIZE_LEAKS m->lsan_tag = __lsan::DisabledInThisThread() ? __lsan::kIgnored : __lsan::kDirectlyLeaked; #endif // Must be the last mutation of metadata in this function. atomic_store((atomic_uint8_t *)m, CHUNK_ALLOCATED, memory_order_release); ASAN_MALLOC_HOOK(res, size); return res; } // Set quarantine flag if chunk is allocated, issue ASan error report on // available and quarantined chunks. Return true on success, false otherwise. bool AtomicallySetQuarantineFlagIfAllocated(AsanChunk *m, void *ptr, BufferedStackTrace *stack) { u8 old_chunk_state = CHUNK_ALLOCATED; // Flip the chunk_state atomically to avoid race on double-free. if (!atomic_compare_exchange_strong((atomic_uint8_t *)m, &old_chunk_state, CHUNK_QUARANTINE, memory_order_acquire)) { ReportInvalidFree(ptr, old_chunk_state, stack); // It's not safe to push a chunk in quarantine on invalid free. return false; } CHECK_EQ(CHUNK_ALLOCATED, old_chunk_state); return true; } // Expects the chunk to already be marked as quarantined by using // AtomicallySetQuarantineFlagIfAllocated. void QuarantineChunk(AsanChunk *m, void *ptr, BufferedStackTrace *stack, AllocType alloc_type) { CHECK_EQ(m->chunk_state, CHUNK_QUARANTINE); CHECK_GE(m->alloc_tid, 0); if (SANITIZER_WORDSIZE == 64) // On 32-bits this resides in user area. CHECK_EQ(m->free_tid, kInvalidTid); AsanThread *t = GetCurrentThread(); m->free_tid = t ? t->tid() : 0; m->free_context_id = StackDepotPut(*stack); // Poison the region. PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), SHADOW_GRANULARITY), kAsanHeapFreeMagic); AsanStats &thread_stats = GetCurrentThreadStats(); thread_stats.frees++; thread_stats.freed += m->UsedSize(); // Push into quarantine. if (t) { AsanThreadLocalMallocStorage *ms = &t->malloc_storage(); AllocatorCache *ac = GetAllocatorCache(ms); quarantine.Put(GetQuarantineCache(ms), QuarantineCallback(ac), m, m->UsedSize()); } else { SpinMutexLock l(&fallback_mutex); AllocatorCache *ac = &fallback_allocator_cache; quarantine.Put(&fallback_quarantine_cache, QuarantineCallback(ac), m, m->UsedSize()); } } void Deallocate(void *ptr, uptr delete_size, BufferedStackTrace *stack, AllocType alloc_type) { uptr p = reinterpret_cast(ptr); if (p == 0) return; uptr chunk_beg = p - kChunkHeaderSize; AsanChunk *m = reinterpret_cast(chunk_beg); ASAN_FREE_HOOK(ptr); // Must mark the chunk as quarantined before any changes to its metadata. // Do not quarantine given chunk if we failed to set CHUNK_QUARANTINE flag. if (!AtomicallySetQuarantineFlagIfAllocated(m, ptr, stack)) return; if (m->alloc_type != alloc_type) { if (atomic_load(&alloc_dealloc_mismatch, memory_order_acquire)) { ReportAllocTypeMismatch((uptr)ptr, stack, (AllocType)m->alloc_type, (AllocType)alloc_type); } } if (delete_size && flags()->new_delete_type_mismatch && delete_size != m->UsedSize()) { ReportNewDeleteSizeMismatch(p, delete_size, stack); } QuarantineChunk(m, ptr, stack, alloc_type); } void *Reallocate(void *old_ptr, uptr new_size, BufferedStackTrace *stack) { CHECK(old_ptr && new_size); uptr p = reinterpret_cast(old_ptr); uptr chunk_beg = p - kChunkHeaderSize; AsanChunk *m = reinterpret_cast(chunk_beg); AsanStats &thread_stats = GetCurrentThreadStats(); thread_stats.reallocs++; thread_stats.realloced += new_size; void *new_ptr = Allocate(new_size, 8, stack, FROM_MALLOC, true); if (new_ptr) { u8 chunk_state = m->chunk_state; if (chunk_state != CHUNK_ALLOCATED) ReportInvalidFree(old_ptr, chunk_state, stack); CHECK_NE(REAL(memcpy), nullptr); uptr memcpy_size = Min(new_size, m->UsedSize()); // If realloc() races with free(), we may start copying freed memory. // However, we will report racy double-free later anyway. REAL(memcpy)(new_ptr, old_ptr, memcpy_size); Deallocate(old_ptr, 0, stack, FROM_MALLOC); } return new_ptr; } void *Calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { if (CallocShouldReturnNullDueToOverflow(size, nmemb)) return allocator.ReturnNullOrDieOnBadRequest(); void *ptr = Allocate(nmemb * size, 8, stack, FROM_MALLOC, false); // If the memory comes from the secondary allocator no need to clear it // as it comes directly from mmap. if (ptr && allocator.FromPrimary(ptr)) REAL(memset)(ptr, 0, nmemb * size); return ptr; } void ReportInvalidFree(void *ptr, u8 chunk_state, BufferedStackTrace *stack) { if (chunk_state == CHUNK_QUARANTINE) ReportDoubleFree((uptr)ptr, stack); else ReportFreeNotMalloced((uptr)ptr, stack); } void CommitBack(AsanThreadLocalMallocStorage *ms) { AllocatorCache *ac = GetAllocatorCache(ms); quarantine.Drain(GetQuarantineCache(ms), QuarantineCallback(ac)); allocator.SwallowCache(ac); } // -------------------------- Chunk lookup ---------------------- // Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg). AsanChunk *GetAsanChunk(void *alloc_beg) { if (!alloc_beg) return nullptr; if (!allocator.FromPrimary(alloc_beg)) { uptr *meta = reinterpret_cast(allocator.GetMetaData(alloc_beg)); AsanChunk *m = reinterpret_cast(meta[1]); return m; } uptr *alloc_magic = reinterpret_cast(alloc_beg); if (alloc_magic[0] == kAllocBegMagic) return reinterpret_cast(alloc_magic[1]); return reinterpret_cast(alloc_beg); } AsanChunk *GetAsanChunkByAddr(uptr p) { void *alloc_beg = allocator.GetBlockBegin(reinterpret_cast(p)); return GetAsanChunk(alloc_beg); } // Allocator must be locked when this function is called. AsanChunk *GetAsanChunkByAddrFastLocked(uptr p) { void *alloc_beg = allocator.GetBlockBeginFastLocked(reinterpret_cast(p)); return GetAsanChunk(alloc_beg); } uptr AllocationSize(uptr p) { AsanChunk *m = GetAsanChunkByAddr(p); if (!m) return 0; if (m->chunk_state != CHUNK_ALLOCATED) return 0; if (m->Beg() != p) return 0; return m->UsedSize(); } AsanChunkView FindHeapChunkByAddress(uptr addr) { AsanChunk *m1 = GetAsanChunkByAddr(addr); if (!m1) return AsanChunkView(m1); sptr offset = 0; if (AsanChunkView(m1).AddrIsAtLeft(addr, 1, &offset)) { // The address is in the chunk's left redzone, so maybe it is actually // a right buffer overflow from the other chunk to the left. // Search a bit to the left to see if there is another chunk. AsanChunk *m2 = nullptr; for (uptr l = 1; l < GetPageSizeCached(); l++) { m2 = GetAsanChunkByAddr(addr - l); if (m2 == m1) continue; // Still the same chunk. break; } if (m2 && AsanChunkView(m2).AddrIsAtRight(addr, 1, &offset)) m1 = ChooseChunk(addr, m2, m1); } return AsanChunkView(m1); } void PrintStats() { allocator.PrintStats(); quarantine.PrintStats(); } void ForceLock() { allocator.ForceLock(); fallback_mutex.Lock(); } void ForceUnlock() { fallback_mutex.Unlock(); allocator.ForceUnlock(); } }; static Allocator instance(LINKER_INITIALIZED); static AsanAllocator &get_allocator() { return instance.allocator; } bool AsanChunkView::IsValid() const { return chunk_ && chunk_->chunk_state != CHUNK_AVAILABLE; } bool AsanChunkView::IsAllocated() const { return chunk_ && chunk_->chunk_state == CHUNK_ALLOCATED; } bool AsanChunkView::IsQuarantined() const { return chunk_ && chunk_->chunk_state == CHUNK_QUARANTINE; } uptr AsanChunkView::Beg() const { return chunk_->Beg(); } uptr AsanChunkView::End() const { return Beg() + UsedSize(); } uptr AsanChunkView::UsedSize() const { return chunk_->UsedSize(); } uptr AsanChunkView::AllocTid() const { return chunk_->alloc_tid; } uptr AsanChunkView::FreeTid() const { return chunk_->free_tid; } AllocType AsanChunkView::GetAllocType() const { return (AllocType)chunk_->alloc_type; } static StackTrace GetStackTraceFromId(u32 id) { CHECK(id); StackTrace res = StackDepotGet(id); CHECK(res.trace); return res; } u32 AsanChunkView::GetAllocStackId() const { return chunk_->alloc_context_id; } u32 AsanChunkView::GetFreeStackId() const { return chunk_->free_context_id; } StackTrace AsanChunkView::GetAllocStack() const { return GetStackTraceFromId(GetAllocStackId()); } StackTrace AsanChunkView::GetFreeStack() const { return GetStackTraceFromId(GetFreeStackId()); } void InitializeAllocator(const AllocatorOptions &options) { instance.Initialize(options); } void ReInitializeAllocator(const AllocatorOptions &options) { instance.ReInitialize(options); } void GetAllocatorOptions(AllocatorOptions *options) { instance.GetOptions(options); } AsanChunkView FindHeapChunkByAddress(uptr addr) { return instance.FindHeapChunkByAddress(addr); } AsanChunkView FindHeapChunkByAllocBeg(uptr addr) { return AsanChunkView(instance.GetAsanChunk(reinterpret_cast(addr))); } void AsanThreadLocalMallocStorage::CommitBack() { instance.CommitBack(this); } void PrintInternalAllocatorStats() { instance.PrintStats(); } void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack, AllocType alloc_type) { return instance.Allocate(size, alignment, stack, alloc_type, true); } void asan_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) { instance.Deallocate(ptr, 0, stack, alloc_type); } void asan_sized_free(void *ptr, uptr size, BufferedStackTrace *stack, AllocType alloc_type) { instance.Deallocate(ptr, size, stack, alloc_type); } void *asan_malloc(uptr size, BufferedStackTrace *stack) { return instance.Allocate(size, 8, stack, FROM_MALLOC, true); } void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { return instance.Calloc(nmemb, size, stack); } void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) { if (!p) return instance.Allocate(size, 8, stack, FROM_MALLOC, true); if (size == 0) { instance.Deallocate(p, 0, stack, FROM_MALLOC); return nullptr; } return instance.Reallocate(p, size, stack); } void *asan_valloc(uptr size, BufferedStackTrace *stack) { return instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC, true); } void *asan_pvalloc(uptr size, BufferedStackTrace *stack) { uptr PageSize = GetPageSizeCached(); size = RoundUpTo(size, PageSize); if (size == 0) { // pvalloc(0) should allocate one page. size = PageSize; } return instance.Allocate(size, PageSize, stack, FROM_MALLOC, true); } int asan_posix_memalign(void **memptr, uptr alignment, uptr size, BufferedStackTrace *stack) { void *ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC, true); CHECK(IsAligned((uptr)ptr, alignment)); *memptr = ptr; return 0; } uptr asan_malloc_usable_size(const void *ptr, uptr pc, uptr bp) { if (!ptr) return 0; uptr usable_size = instance.AllocationSize(reinterpret_cast(ptr)); if (flags()->check_malloc_usable_size && (usable_size == 0)) { GET_STACK_TRACE_FATAL(pc, bp); ReportMallocUsableSizeNotOwned((uptr)ptr, &stack); } return usable_size; } uptr asan_mz_size(const void *ptr) { return instance.AllocationSize(reinterpret_cast(ptr)); } void asan_mz_force_lock() { instance.ForceLock(); } void asan_mz_force_unlock() { instance.ForceUnlock(); } void AsanSoftRssLimitExceededCallback(bool exceeded) { instance.allocator.SetRssLimitIsExceeded(exceeded); } } // namespace __asan // --- Implementation of LSan-specific functions --- {{{1 namespace __lsan { void LockAllocator() { __asan::get_allocator().ForceLock(); } void UnlockAllocator() { __asan::get_allocator().ForceUnlock(); } void GetAllocatorGlobalRange(uptr *begin, uptr *end) { *begin = (uptr)&__asan::get_allocator(); *end = *begin + sizeof(__asan::get_allocator()); } uptr PointsIntoChunk(void* p) { uptr addr = reinterpret_cast(p); __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(addr); if (!m) return 0; uptr chunk = m->Beg(); if (m->chunk_state != __asan::CHUNK_ALLOCATED) return 0; if (m->AddrIsInside(addr, /*locked_version=*/true)) return chunk; if (IsSpecialCaseOfOperatorNew0(chunk, m->UsedSize(/*locked_version*/ true), addr)) return chunk; return 0; } uptr GetUserBegin(uptr chunk) { __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddrFastLocked(chunk); CHECK(m); return m->Beg(); } LsanMetadata::LsanMetadata(uptr chunk) { metadata_ = reinterpret_cast(chunk - __asan::kChunkHeaderSize); } bool LsanMetadata::allocated() const { __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); return m->chunk_state == __asan::CHUNK_ALLOCATED; } ChunkTag LsanMetadata::tag() const { __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); return static_cast(m->lsan_tag); } void LsanMetadata::set_tag(ChunkTag value) { __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); m->lsan_tag = value; } uptr LsanMetadata::requested_size() const { __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); return m->UsedSize(/*locked_version=*/true); } u32 LsanMetadata::stack_trace_id() const { __asan::AsanChunk *m = reinterpret_cast<__asan::AsanChunk *>(metadata_); return m->alloc_context_id; } void ForEachChunk(ForEachChunkCallback callback, void *arg) { __asan::get_allocator().ForEachChunk(callback, arg); } IgnoreObjectResult IgnoreObjectLocked(const void *p) { uptr addr = reinterpret_cast(p); __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddr(addr); if (!m) return kIgnoreObjectInvalid; if ((m->chunk_state == __asan::CHUNK_ALLOCATED) && m->AddrIsInside(addr)) { if (m->lsan_tag == kIgnored) return kIgnoreObjectAlreadyIgnored; m->lsan_tag = __lsan::kIgnored; return kIgnoreObjectSuccess; } else { return kIgnoreObjectInvalid; } } } // namespace __lsan // ---------------------- Interface ---------------- {{{1 using namespace __asan; // NOLINT // ASan allocator doesn't reserve extra bytes, so normally we would // just return "size". We don't want to expose our redzone sizes, etc here. uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } int __sanitizer_get_ownership(const void *p) { uptr ptr = reinterpret_cast(p); return instance.AllocationSize(ptr) > 0; } uptr __sanitizer_get_allocated_size(const void *p) { if (!p) return 0; uptr ptr = reinterpret_cast(p); uptr allocated_size = instance.AllocationSize(ptr); // Die if p is not malloced or if it is already freed. if (allocated_size == 0) { GET_STACK_TRACE_FATAL_HERE; ReportSanitizerGetAllocatedSizeNotOwned(ptr, &stack); } return allocated_size; } #if !SANITIZER_SUPPORTS_WEAK_HOOKS // Provide default (no-op) implementation of malloc hooks. extern "C" { SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_malloc_hook(void *ptr, uptr size) { (void)ptr; (void)size; } SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_free_hook(void *ptr) { (void)ptr; } } // extern "C" #endif Index: projects/clang400-import/contrib/compiler-rt/lib/asan/asan_flags.cc =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/asan/asan_flags.cc (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/asan/asan_flags.cc (revision 312198) @@ -1,189 +1,194 @@ //===-- asan_flags.cc -------------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file is a part of AddressSanitizer, an address sanity checker. // // ASan flag parsing logic. //===----------------------------------------------------------------------===// #include "asan_activation.h" #include "asan_flags.h" #include "asan_interface_internal.h" #include "asan_stack.h" #include "lsan/lsan_common.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_flag_parser.h" #include "ubsan/ubsan_flags.h" #include "ubsan/ubsan_platform.h" namespace __asan { Flags asan_flags_dont_use_directly; // use via flags(). static const char *MaybeCallAsanDefaultOptions() { return (&__asan_default_options) ? __asan_default_options() : ""; } static const char *MaybeUseAsanDefaultOptionsCompileDefinition() { #ifdef ASAN_DEFAULT_OPTIONS // Stringize the macro value. # define ASAN_STRINGIZE(x) #x # define ASAN_STRINGIZE_OPTIONS(options) ASAN_STRINGIZE(options) return ASAN_STRINGIZE_OPTIONS(ASAN_DEFAULT_OPTIONS); #else return ""; #endif } void Flags::SetDefaults() { #define ASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; #include "asan_flags.inc" #undef ASAN_FLAG } static void RegisterAsanFlags(FlagParser *parser, Flags *f) { #define ASAN_FLAG(Type, Name, DefaultValue, Description) \ RegisterFlag(parser, #Name, Description, &f->Name); #include "asan_flags.inc" #undef ASAN_FLAG } void InitializeFlags() { // Set the default values and prepare for parsing ASan and common flags. SetCommonFlagsDefaults(); { CommonFlags cf; cf.CopyFrom(*common_flags()); cf.detect_leaks = CAN_SANITIZE_LEAKS; cf.external_symbolizer_path = GetEnv("ASAN_SYMBOLIZER_PATH"); cf.malloc_context_size = kDefaultMallocContextSize; cf.intercept_tls_get_addr = true; cf.exitcode = 1; OverrideCommonFlags(cf); } Flags *f = flags(); f->SetDefaults(); FlagParser asan_parser; RegisterAsanFlags(&asan_parser, f); RegisterCommonFlags(&asan_parser); // Set the default values and prepare for parsing LSan and UBSan flags // (which can also overwrite common flags). #if CAN_SANITIZE_LEAKS __lsan::Flags *lf = __lsan::flags(); lf->SetDefaults(); FlagParser lsan_parser; __lsan::RegisterLsanFlags(&lsan_parser, lf); RegisterCommonFlags(&lsan_parser); #endif #if CAN_SANITIZE_UB __ubsan::Flags *uf = __ubsan::flags(); uf->SetDefaults(); FlagParser ubsan_parser; __ubsan::RegisterUbsanFlags(&ubsan_parser, uf); RegisterCommonFlags(&ubsan_parser); #endif // Override from ASan compile definition. const char *asan_compile_def = MaybeUseAsanDefaultOptionsCompileDefinition(); asan_parser.ParseString(asan_compile_def); // Override from user-specified string. const char *asan_default_options = MaybeCallAsanDefaultOptions(); asan_parser.ParseString(asan_default_options); #if CAN_SANITIZE_UB const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions(); ubsan_parser.ParseString(ubsan_default_options); #endif // Override from command line. asan_parser.ParseString(GetEnv("ASAN_OPTIONS")); #if CAN_SANITIZE_LEAKS lsan_parser.ParseString(GetEnv("LSAN_OPTIONS")); #endif #if CAN_SANITIZE_UB ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS")); #endif InitializeCommonFlags(); // TODO(eugenis): dump all flags at verbosity>=2? if (Verbosity()) ReportUnrecognizedFlags(); if (common_flags()->help) { // TODO(samsonov): print all of the flags (ASan, LSan, common). asan_parser.PrintFlagDescriptions(); } // Flag validation: if (!CAN_SANITIZE_LEAKS && common_flags()->detect_leaks) { Report("%s: detect_leaks is not supported on this platform.\n", SanitizerToolName); Die(); } // Make "strict_init_order" imply "check_initialization_order". // TODO(samsonov): Use a single runtime flag for an init-order checker. if (f->strict_init_order) { f->check_initialization_order = true; } CHECK_LE((uptr)common_flags()->malloc_context_size, kStackTraceMax); CHECK_LE(f->min_uar_stack_size_log, f->max_uar_stack_size_log); CHECK_GE(f->redzone, 16); CHECK_GE(f->max_redzone, f->redzone); CHECK_LE(f->max_redzone, 2048); CHECK(IsPowerOfTwo(f->redzone)); CHECK(IsPowerOfTwo(f->max_redzone)); // quarantine_size is deprecated but we still honor it. // quarantine_size can not be used together with quarantine_size_mb. if (f->quarantine_size >= 0 && f->quarantine_size_mb >= 0) { Report("%s: please use either 'quarantine_size' (deprecated) or " "quarantine_size_mb, but not both\n", SanitizerToolName); Die(); } if (f->quarantine_size >= 0) f->quarantine_size_mb = f->quarantine_size >> 20; if (f->quarantine_size_mb < 0) { const int kDefaultQuarantineSizeMb = (ASAN_LOW_MEMORY) ? 1UL << 4 : 1UL << 8; f->quarantine_size_mb = kDefaultQuarantineSizeMb; } if (f->thread_local_quarantine_size_kb < 0) { const u32 kDefaultThreadLocalQuarantineSizeKb = // It is not advised to go lower than 64Kb, otherwise quarantine batches // pushed from thread local quarantine to global one will create too // much overhead. One quarantine batch size is 8Kb and it holds up to // 1021 chunk, which amounts to 1/8 memory overhead per batch when // thread local quarantine is set to 64Kb. (ASAN_LOW_MEMORY) ? 1 << 6 : FIRST_32_SECOND_64(1 << 8, 1 << 10); f->thread_local_quarantine_size_kb = kDefaultThreadLocalQuarantineSizeKb; } + if (f->thread_local_quarantine_size_kb == 0 && f->quarantine_size_mb > 0) { + Report("%s: thread_local_quarantine_size_kb can be set to 0 only when " + "quarantine_size_mb is set to 0\n", SanitizerToolName); + Die(); + } if (!f->replace_str && common_flags()->intercept_strlen) { Report("WARNING: strlen interceptor is enabled even though replace_str=0. " "Use intercept_strlen=0 to disable it."); } if (!f->replace_str && common_flags()->intercept_strchr) { Report("WARNING: strchr* interceptors are enabled even though " "replace_str=0. Use intercept_strchr=0 to disable them."); } } } // namespace __asan #if !SANITIZER_SUPPORTS_WEAK_HOOKS extern "C" { SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char* __asan_default_options() { return ""; } } // extern "C" #endif Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/adddf3vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/adddf3vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/adddf3vfp.S (revision 312198) @@ -1,29 +1,33 @@ //===-- adddf3vfp.S - Implement adddf3vfp ---------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // double __adddf3vfp(double a, double b) { return a + b; } // // Adds two double precision floating point numbers using the Darwin // calling convention where double arguments are passsed in GPR pairs // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__adddf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vadd.f64 d0, d0, d1 +#else vmov d6, r0, r1 // move first param from r0/r1 pair into d6 vmov d7, r2, r3 // move second param from r2/r3 pair into d7 vadd.f64 d6, d6, d7 vmov r0, r1, d6 // move result back to r0/r1 pair +#endif bx lr END_COMPILERRT_FUNCTION(__adddf3vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/addsf3vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/addsf3vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/addsf3vfp.S (revision 312198) @@ -1,29 +1,33 @@ //===-- addsf3vfp.S - Implement addsf3vfp ---------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern float __addsf3vfp(float a, float b); // // Adds two single precision floating point numbers using the Darwin // calling convention where single arguments are passsed in GPRs // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__addsf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vadd.f32 s0, s0, s1 +#else vmov s14, r0 // move first param from r0 into float register vmov s15, r1 // move second param from r1 into float register vadd.f32 s14, s14, s15 vmov r0, s14 // move result back to r0 +#endif bx lr END_COMPILERRT_FUNCTION(__addsf3vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/comparesf2.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/comparesf2.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/comparesf2.S (revision 312198) @@ -1,266 +1,296 @@ //===-- comparesf2.S - Implement single-precision soft-float comparisons --===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file implements the following soft-fp_t comparison routines: // // __eqsf2 __gesf2 __unordsf2 // __lesf2 __gtsf2 // __ltsf2 // __nesf2 // // The semantics of the routines grouped in each column are identical, so there // is a single implementation for each, with multiple names. // // The routines behave as follows: // // __lesf2(a,b) returns -1 if a < b // 0 if a == b // 1 if a > b // 1 if either a or b is NaN // // __gesf2(a,b) returns -1 if a < b // 0 if a == b // 1 if a > b // -1 if either a or b is NaN // // __unordsf2(a,b) returns 0 if both a and b are numbers // 1 if either a or b is NaN // // Note that __lesf2( ) and __gesf2( ) are identical except in their handling of // NaN values. // //===----------------------------------------------------------------------===// #include "../assembly.h" .syntax unified #if __ARM_ARCH_ISA_THUMB == 2 .thumb #endif -.p2align 2 +@ int __eqsf2(float a, float b) + + .p2align 2 DEFINE_COMPILERRT_FUNCTION(__eqsf2) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmov r0, s0 + vmov r1, s1 +#endif // Make copies of a and b with the sign bit shifted off the top. These will // be used to detect zeros and NaNs. #if __ARM_ARCH_ISA_THUMB == 1 push {r6, lr} lsls r2, r0, #1 lsls r3, r1, #1 #else mov r2, r0, lsl #1 mov r3, r1, lsl #1 #endif // We do the comparison in three stages (ignoring NaN values for the time // being). First, we orr the absolute values of a and b; this sets the Z // flag if both a and b are zero (of either sign). The shift of r3 doesn't // effect this at all, but it *does* make sure that the C flag is clear for // the subsequent operations. #if __ARM_ARCH_ISA_THUMB == 1 lsrs r6, r3, #1 orrs r6, r2, r6 #else orrs r12, r2, r3, lsr #1 #endif // Next, we check if a and b have the same or different signs. If they have // opposite signs, this eor will set the N flag. #if __ARM_ARCH_ISA_THUMB == 1 beq 1f movs r6, r0 eors r6, r1 1: #else it ne eorsne r12, r0, r1 #endif // If a and b are equal (either both zeros or bit identical; again, we're // ignoring NaNs for now), this subtract will zero out r0. If they have the // same sign, the flags are updated as they would be for a comparison of the // absolute values of a and b. #if __ARM_ARCH_ISA_THUMB == 1 bmi 1f subs r0, r2, r3 1: #else it pl subspl r0, r2, r3 #endif // If a is smaller in magnitude than b and both have the same sign, place // the negation of the sign of b in r0. Thus, if both are negative and // a > b, this sets r0 to 0; if both are positive and a < b, this sets // r0 to -1. // // This is also done if a and b have opposite signs and are not both zero, // because in that case the subtract was not performed and the C flag is // still clear from the shift argument in orrs; if a is positive and b // negative, this places 0 in r0; if a is negative and b positive, -1 is // placed in r0. #if __ARM_ARCH_ISA_THUMB == 1 bhs 1f // Here if a and b have the same sign and absA < absB, the result is thus // b < 0 ? 1 : -1. Same if a and b have the opposite sign (ignoring Nan). movs r0, #1 lsrs r1, #31 bne LOCAL_LABEL(CHECK_NAN) negs r0, r0 b LOCAL_LABEL(CHECK_NAN) 1: #else it lo mvnlo r0, r1, asr #31 #endif // If a is greater in magnitude than b and both have the same sign, place // the sign of b in r0. Thus, if both are negative and a < b, -1 is placed // in r0, which is the desired result. Conversely, if both are positive // and a > b, zero is placed in r0. #if __ARM_ARCH_ISA_THUMB == 1 bls 1f // Here both have the same sign and absA > absB. movs r0, #1 lsrs r1, #31 beq LOCAL_LABEL(CHECK_NAN) negs r0, r0 1: #else it hi movhi r0, r1, asr #31 #endif // If you've been keeping track, at this point r0 contains -1 if a < b and // 0 if a >= b. All that remains to be done is to set it to 1 if a > b. // If a == b, then the Z flag is set, so we can get the correct final value // into r0 by simply or'ing with 1 if Z is clear. // For Thumb-1, r0 contains -1 if a < b, 0 if a > b and 0 if a == b. #if __ARM_ARCH_ISA_THUMB != 1 it ne orrne r0, r0, #1 #endif // Finally, we need to deal with NaNs. If either argument is NaN, replace // the value in r0 with 1. #if __ARM_ARCH_ISA_THUMB == 1 LOCAL_LABEL(CHECK_NAN): movs r6, #0xff lsls r6, #24 cmp r2, r6 bhi 1f cmp r3, r6 1: bls 2f movs r0, #1 2: pop {r6, pc} #else cmp r2, #0xff000000 ite ls cmpls r3, #0xff000000 movhi r0, #1 JMP(lr) #endif END_COMPILERRT_FUNCTION(__eqsf2) + DEFINE_COMPILERRT_FUNCTION_ALIAS(__lesf2, __eqsf2) DEFINE_COMPILERRT_FUNCTION_ALIAS(__ltsf2, __eqsf2) DEFINE_COMPILERRT_FUNCTION_ALIAS(__nesf2, __eqsf2) -.p2align 2 +@ int __gtsf2(float a, float b) + + .p2align 2 DEFINE_COMPILERRT_FUNCTION(__gtsf2) // Identical to the preceding except in that we return -1 for NaN values. // Given that the two paths share so much code, one might be tempted to // unify them; however, the extra code needed to do so makes the code size // to performance tradeoff very hard to justify for such small functions. +#if defined(COMPILER_RT_ARMHF_TARGET) + vmov r0, s0 + vmov r1, s1 +#endif #if __ARM_ARCH_ISA_THUMB == 1 push {r6, lr} lsls r2, r0, #1 lsls r3, r1, #1 lsrs r6, r3, #1 orrs r6, r2, r6 beq 1f movs r6, r0 eors r6, r1 1: bmi 2f subs r0, r2, r3 2: bhs 3f movs r0, #1 lsrs r1, #31 bne LOCAL_LABEL(CHECK_NAN_2) negs r0, r0 b LOCAL_LABEL(CHECK_NAN_2) 3: bls 4f movs r0, #1 lsrs r1, #31 beq LOCAL_LABEL(CHECK_NAN_2) negs r0, r0 4: LOCAL_LABEL(CHECK_NAN_2): movs r6, #0xff lsls r6, #24 cmp r2, r6 bhi 5f cmp r3, r6 5: bls 6f movs r0, #1 negs r0, r0 6: pop {r6, pc} #else + mov r2, r0, lsl #1 + mov r3, r1, lsl #1 orrs r12, r2, r3, lsr #1 it ne eorsne r12, r0, r1 it pl subspl r0, r2, r3 it lo mvnlo r0, r1, asr #31 it hi movhi r0, r1, asr #31 it ne orrne r0, r0, #1 cmp r2, #0xff000000 ite ls cmpls r3, #0xff000000 movhi r0, #-1 JMP(lr) #endif END_COMPILERRT_FUNCTION(__gtsf2) + DEFINE_COMPILERRT_FUNCTION_ALIAS(__gesf2, __gtsf2) -.p2align 2 +@ int __unordsf2(float a, float b) + + .p2align 2 DEFINE_COMPILERRT_FUNCTION(__unordsf2) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmov r0, s0 + vmov r1, s1 +#endif // Return 1 for NaN values, 0 otherwise. lsls r2, r0, #1 lsls r3, r1, #1 movs r0, #0 #if __ARM_ARCH_ISA_THUMB == 1 movs r1, #0xff lsls r1, #24 cmp r2, r1 bhi 1f cmp r3, r1 1: bls 2f movs r0, #1 2: #else cmp r2, #0xff000000 ite ls cmpls r3, #0xff000000 movhi r0, #1 #endif JMP(lr) END_COMPILERRT_FUNCTION(__unordsf2) +#if defined(COMPILER_RT_ARMHF_TARGET) +DEFINE_COMPILERRT_FUNCTION(__aeabi_fcmpum): + vmov s0, r0 + vmov s1, r1 + b SYMBOL_NAME(__unordsf2) +END_COMPILERRT_FUNCTION(__aeabi_fcmpum) +#else DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_fcmpun, __unordsf2) +#endif NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/divdf3vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/divdf3vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/divdf3vfp.S (revision 312198) @@ -1,29 +1,33 @@ //===-- divdf3vfp.S - Implement divdf3vfp ---------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern double __divdf3vfp(double a, double b); // // Divides two double precision floating point numbers using the Darwin // calling convention where double arguments are passsed in GPR pairs // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__divdf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vdiv.f64 d0, d0, d1 +#else vmov d6, r0, r1 // move first param from r0/r1 pair into d6 vmov d7, r2, r3 // move second param from r2/r3 pair into d7 - vdiv.f64 d5, d6, d7 + vdiv.f64 d5, d6, d7 vmov r0, r1, d5 // move result back to r0/r1 pair +#endif bx lr END_COMPILERRT_FUNCTION(__divdf3vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/divsf3vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/divsf3vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/divsf3vfp.S (revision 312198) @@ -1,29 +1,33 @@ //===-- divsf3vfp.S - Implement divsf3vfp ---------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern float __divsf3vfp(float a, float b); // // Divides two single precision floating point numbers using the Darwin // calling convention where single arguments are passsed like 32-bit ints. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__divsf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vdiv.f32 s0, s0, s1 +#else vmov s14, r0 // move first param from r0 into float register vmov s15, r1 // move second param from r1 into float register vdiv.f32 s13, s14, s15 vmov r0, s13 // move result back to r0 +#endif bx lr END_COMPILERRT_FUNCTION(__divsf3vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/eqdf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/eqdf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/eqdf2vfp.S (revision 312198) @@ -1,32 +1,36 @@ //===-- eqdf2vfp.S - Implement eqdf2vfp -----------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern int __eqdf2vfp(double a, double b); // // Returns one iff a == b and neither is NaN. // Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__eqdf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f64 d0, d1 +#else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register vcmp.f64 d6, d7 +#endif vmrs apsr_nzcv, fpscr moveq r0, #1 // set result register to 1 if equal movne r0, #0 bx lr END_COMPILERRT_FUNCTION(__eqdf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/eqsf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/eqsf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/eqsf2vfp.S (revision 312198) @@ -1,32 +1,36 @@ //===-- eqsf2vfp.S - Implement eqsf2vfp -----------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern int __eqsf2vfp(float a, float b); // // Returns one iff a == b and neither is NaN. // Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__eqsf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f32 s0, s1 +#else vmov s14, r0 // move from GPR 0 to float register vmov s15, r1 // move from GPR 1 to float register vcmp.f32 s14, s15 +#endif vmrs apsr_nzcv, fpscr moveq r0, #1 // set result register to 1 if equal movne r0, #0 bx lr END_COMPILERRT_FUNCTION(__eqsf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/extendsfdf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/extendsfdf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/extendsfdf2vfp.S (revision 312198) @@ -1,29 +1,33 @@ //===-- extendsfdf2vfp.S - Implement extendsfdf2vfp -----------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern double __extendsfdf2vfp(float a); // // Converts single precision float to double precision result. // Uses Darwin calling convention where a single precision parameter is // passed in a GPR and a double precision result is returned in R0/R1 pair. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__extendsfdf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcvt.f64.f32 d0, s0 +#else vmov s15, r0 // load float register from R0 vcvt.f64.f32 d7, s15 // convert single to double vmov r0, r1, d7 // return result in r0/r1 pair +#endif bx lr END_COMPILERRT_FUNCTION(__extendsfdf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/fixdfsivfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/fixdfsivfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/fixdfsivfp.S (revision 312198) @@ -1,29 +1,34 @@ //===-- fixdfsivfp.S - Implement fixdfsivfp -----------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern int __fixdfsivfp(double a); // // Converts double precision float to a 32-bit int rounding towards zero. // Uses Darwin calling convention where a double precision parameter is // passed in GPR register pair. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__fixdfsivfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcvt.s32.f64 s0, d0 + vmov r0, s0 +#else vmov d7, r0, r1 // load double register from R0/R1 vcvt.s32.f64 s15, d7 // convert double to 32-bit int into s15 vmov r0, s15 // move s15 to result register +#endif bx lr END_COMPILERRT_FUNCTION(__fixdfsivfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/fixsfsivfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/fixsfsivfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/fixsfsivfp.S (revision 312198) @@ -1,29 +1,34 @@ //===-- fixsfsivfp.S - Implement fixsfsivfp -----------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern int __fixsfsivfp(float a); // // Converts single precision float to a 32-bit int rounding towards zero. // Uses Darwin calling convention where a single precision parameter is // passed in a GPR.. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__fixsfsivfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcvt.s32.f32 s0, s0 + vmov r0, s0 +#else vmov s15, r0 // load float register from R0 vcvt.s32.f32 s15, s15 // convert single to 32-bit int into s15 vmov r0, s15 // move s15 to result register +#endif bx lr END_COMPILERRT_FUNCTION(__fixsfsivfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/fixunsdfsivfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/fixunsdfsivfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/fixunsdfsivfp.S (revision 312198) @@ -1,30 +1,35 @@ //===-- fixunsdfsivfp.S - Implement fixunsdfsivfp -------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern unsigned int __fixunsdfsivfp(double a); // // Converts double precision float to a 32-bit unsigned int rounding towards // zero. All negative values become zero. // Uses Darwin calling convention where a double precision parameter is // passed in GPR register pair. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__fixunsdfsivfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcvt.u32.f64 s0, d0 + vmov r0, s0 +#else vmov d7, r0, r1 // load double register from R0/R1 vcvt.u32.f64 s15, d7 // convert double to 32-bit int into s15 vmov r0, s15 // move s15 to result register +#endif bx lr END_COMPILERRT_FUNCTION(__fixunsdfsivfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/fixunssfsivfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/fixunssfsivfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/fixunssfsivfp.S (revision 312198) @@ -1,30 +1,35 @@ //===-- fixunssfsivfp.S - Implement fixunssfsivfp -------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern unsigned int __fixunssfsivfp(float a); // // Converts single precision float to a 32-bit unsigned int rounding towards // zero. All negative values become zero. // Uses Darwin calling convention where a single precision parameter is // passed in a GPR.. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__fixunssfsivfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcvt.u32.f32 s0, s0 + vmov r0, s0 +#else vmov s15, r0 // load float register from R0 vcvt.u32.f32 s15, s15 // convert single to 32-bit unsigned into s15 vmov r0, s15 // move s15 to result register +#endif bx lr END_COMPILERRT_FUNCTION(__fixunssfsivfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/floatsidfvfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/floatsidfvfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/floatsidfvfp.S (revision 312198) @@ -1,29 +1,34 @@ //===-- floatsidfvfp.S - Implement floatsidfvfp ---------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern double __floatsidfvfp(int a); // // Converts a 32-bit int to a double precision float. // Uses Darwin calling convention where a double precision result is // return in GPR register pair. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__floatsidfvfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmov s0, r0 + vcvt.f64.s32 d0, s0 +#else vmov s15, r0 // move int to float register s15 vcvt.f64.s32 d7, s15 // convert 32-bit int in s15 to double in d7 vmov r0, r1, d7 // move d7 to result register pair r0/r1 +#endif bx lr END_COMPILERRT_FUNCTION(__floatsidfvfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/floatsisfvfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/floatsisfvfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/floatsisfvfp.S (revision 312198) @@ -1,29 +1,34 @@ //===-- floatsisfvfp.S - Implement floatsisfvfp ---------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern float __floatsisfvfp(int a); // // Converts single precision float to a 32-bit int rounding towards zero. // Uses Darwin calling convention where a single precision result is // return in a GPR.. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__floatsisfvfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmov s0, r0 + vcvt.f32.s32 s0, s0 +#else vmov s15, r0 // move int to float register s15 vcvt.f32.s32 s15, s15 // convert 32-bit int in s15 to float in s15 vmov r0, s15 // move s15 to result register +#endif bx lr END_COMPILERRT_FUNCTION(__floatsisfvfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/floatunssidfvfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/floatunssidfvfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/floatunssidfvfp.S (revision 312198) @@ -1,29 +1,34 @@ //===-- floatunssidfvfp.S - Implement floatunssidfvfp ---------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern double __floatunssidfvfp(unsigned int a); // // Converts a 32-bit int to a double precision float. // Uses Darwin calling convention where a double precision result is // return in GPR register pair. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__floatunssidfvfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmov s0, r0 + vcvt.f64.u32 d0, s0 +#else vmov s15, r0 // move int to float register s15 vcvt.f64.u32 d7, s15 // convert 32-bit int in s15 to double in d7 vmov r0, r1, d7 // move d7 to result register pair r0/r1 +#endif bx lr END_COMPILERRT_FUNCTION(__floatunssidfvfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/floatunssisfvfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/floatunssisfvfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/floatunssisfvfp.S (revision 312198) @@ -1,29 +1,34 @@ //===-- floatunssisfvfp.S - Implement floatunssisfvfp ---------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern float __floatunssisfvfp(unsigned int a); // // Converts single precision float to a 32-bit int rounding towards zero. // Uses Darwin calling convention where a single precision result is // return in a GPR.. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__floatunssisfvfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmov s0, r0 + vcvt.f32.u32 s0, s0 +#else vmov s15, r0 // move int to float register s15 vcvt.f32.u32 s15, s15 // convert 32-bit int in s15 to float in s15 vmov r0, s15 // move s15 to result register +#endif bx lr END_COMPILERRT_FUNCTION(__floatunssisfvfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/gedf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/gedf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/gedf2vfp.S (revision 312198) @@ -1,32 +1,36 @@ //===-- gedf2vfp.S - Implement gedf2vfp -----------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern int __gedf2vfp(double a, double b); // // Returns one iff a >= b and neither is NaN. // Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__gedf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f64 d0, d1 +#else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register vcmp.f64 d6, d7 +#endif vmrs apsr_nzcv, fpscr movge r0, #1 // set result register to 1 if greater than or equal movlt r0, #0 bx lr END_COMPILERRT_FUNCTION(__gedf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/gesf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/gesf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/gesf2vfp.S (revision 312198) @@ -1,32 +1,36 @@ //===-- gesf2vfp.S - Implement gesf2vfp -----------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern int __gesf2vfp(float a, float b); // // Returns one iff a >= b and neither is NaN. // Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__gesf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f32 s0, s1 +#else vmov s14, r0 // move from GPR 0 to float register vmov s15, r1 // move from GPR 1 to float register vcmp.f32 s14, s15 +#endif vmrs apsr_nzcv, fpscr movge r0, #1 // set result register to 1 if greater than or equal movlt r0, #0 bx lr END_COMPILERRT_FUNCTION(__gesf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/gtdf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/gtdf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/gtdf2vfp.S (revision 312198) @@ -1,32 +1,36 @@ //===-- gtdf2vfp.S - Implement gtdf2vfp -----------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern double __gtdf2vfp(double a, double b); // // Returns one iff a > b and neither is NaN. // Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__gtdf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f64 d0, d1 +#else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register vcmp.f64 d6, d7 +#endif vmrs apsr_nzcv, fpscr movgt r0, #1 // set result register to 1 if equal movle r0, #0 bx lr END_COMPILERRT_FUNCTION(__gtdf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/gtsf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/gtsf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/gtsf2vfp.S (revision 312198) @@ -1,32 +1,36 @@ //===-- gtsf2vfp.S - Implement gtsf2vfp -----------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern int __gtsf2vfp(float a, float b); // // Returns one iff a > b and neither is NaN. // Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__gtsf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f32 s0, s1 +#else vmov s14, r0 // move from GPR 0 to float register vmov s15, r1 // move from GPR 1 to float register vcmp.f32 s14, s15 +#endif vmrs apsr_nzcv, fpscr movgt r0, #1 // set result register to 1 if equal movle r0, #0 bx lr END_COMPILERRT_FUNCTION(__gtsf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/ledf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/ledf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/ledf2vfp.S (revision 312198) @@ -1,32 +1,36 @@ //===-- ledf2vfp.S - Implement ledf2vfp -----------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern double __ledf2vfp(double a, double b); // // Returns one iff a <= b and neither is NaN. // Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__ledf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f64 d0, d1 +#else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register vcmp.f64 d6, d7 +#endif vmrs apsr_nzcv, fpscr movls r0, #1 // set result register to 1 if equal movhi r0, #0 bx lr END_COMPILERRT_FUNCTION(__ledf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/lesf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/lesf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/lesf2vfp.S (revision 312198) @@ -1,32 +1,36 @@ //===-- lesf2vfp.S - Implement lesf2vfp -----------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern int __lesf2vfp(float a, float b); // // Returns one iff a <= b and neither is NaN. // Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__lesf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f32 s0, s1 +#else vmov s14, r0 // move from GPR 0 to float register vmov s15, r1 // move from GPR 1 to float register vcmp.f32 s14, s15 +#endif vmrs apsr_nzcv, fpscr movls r0, #1 // set result register to 1 if equal movhi r0, #0 bx lr END_COMPILERRT_FUNCTION(__lesf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/ltdf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/ltdf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/ltdf2vfp.S (revision 312198) @@ -1,32 +1,36 @@ //===-- ltdf2vfp.S - Implement ltdf2vfp -----------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern double __ltdf2vfp(double a, double b); // // Returns one iff a < b and neither is NaN. // Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__ltdf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f64 d0, d1 +#else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register vcmp.f64 d6, d7 +#endif vmrs apsr_nzcv, fpscr movmi r0, #1 // set result register to 1 if equal movpl r0, #0 bx lr END_COMPILERRT_FUNCTION(__ltdf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/ltsf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/ltsf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/ltsf2vfp.S (revision 312198) @@ -1,32 +1,36 @@ //===-- ltsf2vfp.S - Implement ltsf2vfp -----------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern int __ltsf2vfp(float a, float b); // // Returns one iff a < b and neither is NaN. // Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__ltsf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f32 s0, s1 +#else vmov s14, r0 // move from GPR 0 to float register vmov s15, r1 // move from GPR 1 to float register vcmp.f32 s14, s15 +#endif vmrs apsr_nzcv, fpscr movmi r0, #1 // set result register to 1 if equal movpl r0, #0 bx lr END_COMPILERRT_FUNCTION(__ltsf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/muldf3vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/muldf3vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/muldf3vfp.S (revision 312198) @@ -1,29 +1,33 @@ //===-- muldf3vfp.S - Implement muldf3vfp ---------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern double __muldf3vfp(double a, double b); // // Multiplies two double precision floating point numbers using the Darwin // calling convention where double arguments are passsed in GPR pairs // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__muldf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmul.f64 d0, d0, d1 +#else vmov d6, r0, r1 // move first param from r0/r1 pair into d6 vmov d7, r2, r3 // move second param from r2/r3 pair into d7 - vmul.f64 d6, d6, d7 + vmul.f64 d6, d6, d7 vmov r0, r1, d6 // move result back to r0/r1 pair +#endif bx lr END_COMPILERRT_FUNCTION(__muldf3vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/mulsf3vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/mulsf3vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/mulsf3vfp.S (revision 312198) @@ -1,29 +1,33 @@ //===-- mulsf3vfp.S - Implement mulsf3vfp ---------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern float __mulsf3vfp(float a, float b); // // Multiplies two single precision floating point numbers using the Darwin // calling convention where single arguments are passsed like 32-bit ints. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__mulsf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmul.f32 s0, s0, s1 +#else vmov s14, r0 // move first param from r0 into float register vmov s15, r1 // move second param from r1 into float register vmul.f32 s13, s14, s15 +#endif vmov r0, s13 // move result back to r0 bx lr END_COMPILERRT_FUNCTION(__mulsf3vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/nedf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/nedf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/nedf2vfp.S (revision 312198) @@ -1,32 +1,36 @@ //===-- nedf2vfp.S - Implement nedf2vfp -----------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern double __nedf2vfp(double a, double b); // // Returns zero if a and b are unequal and neither is NaN. // Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__nedf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f64 d0, d1 +#else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register vcmp.f64 d6, d7 +#endif vmrs apsr_nzcv, fpscr movne r0, #1 // set result register to 0 if unequal moveq r0, #0 bx lr END_COMPILERRT_FUNCTION(__nedf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/negdf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/negdf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/negdf2vfp.S (revision 312198) @@ -1,26 +1,30 @@ //===-- negdf2vfp.S - Implement negdf2vfp ---------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern double __negdf2vfp(double a, double b); // // Returns the negation a double precision floating point numbers using the // Darwin calling convention where double arguments are passsed in GPR pairs. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__negdf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vneg.f64 d0, d0 +#else eor r1, r1, #-2147483648 // flip sign bit on double in r0/r1 pair +#endif bx lr END_COMPILERRT_FUNCTION(__negdf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/negsf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/negsf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/negsf2vfp.S (revision 312198) @@ -1,26 +1,30 @@ //===-- negsf2vfp.S - Implement negsf2vfp ---------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern float __negsf2vfp(float a); // // Returns the negation of a single precision floating point numbers using the // Darwin calling convention where single arguments are passsed like 32-bit ints // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__negsf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vneg.f32 s0, s0 +#else eor r0, r0, #-2147483648 // flip sign bit on float in r0 +#endif bx lr END_COMPILERRT_FUNCTION(__negsf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/nesf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/nesf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/nesf2vfp.S (revision 312198) @@ -1,32 +1,36 @@ //===-- nesf2vfp.S - Implement nesf2vfp -----------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern int __nesf2vfp(float a, float b); // // Returns one iff a != b and neither is NaN. // Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__nesf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f32 s0, s1 +#else vmov s14, r0 // move from GPR 0 to float register vmov s15, r1 // move from GPR 1 to float register vcmp.f32 s14, s15 +#endif vmrs apsr_nzcv, fpscr movne r0, #1 // set result register to 1 if unequal moveq r0, #0 bx lr END_COMPILERRT_FUNCTION(__nesf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/subdf3vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/subdf3vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/subdf3vfp.S (revision 312198) @@ -1,29 +1,33 @@ //===-- subdf3vfp.S - Implement subdf3vfp ---------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern double __subdf3vfp(double a, double b); // // Returns difference between two double precision floating point numbers using // the Darwin calling convention where double arguments are passsed in GPR pairs // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__subdf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vsub.f64 d0, d0, d1 +#else vmov d6, r0, r1 // move first param from r0/r1 pair into d6 vmov d7, r2, r3 // move second param from r2/r3 pair into d7 vsub.f64 d6, d6, d7 vmov r0, r1, d6 // move result back to r0/r1 pair +#endif bx lr END_COMPILERRT_FUNCTION(__subdf3vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/subsf3vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/subsf3vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/subsf3vfp.S (revision 312198) @@ -1,30 +1,34 @@ //===-- subsf3vfp.S - Implement subsf3vfp ---------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern float __subsf3vfp(float a, float b); // -// Returns the difference between two single precision floating point numbers +// Returns the difference between two single precision floating point numbers // using the Darwin calling convention where single arguments are passsed // like 32-bit ints. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__subsf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vsub.f32 s0, s0, s1 +#elsee vmov s14, r0 // move first param from r0 into float register vmov s15, r1 // move second param from r1 into float register vsub.f32 s14, s14, s15 vmov r0, s14 // move result back to r0 +#endif bx lr END_COMPILERRT_FUNCTION(__subsf3vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/truncdfsf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/truncdfsf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/truncdfsf2vfp.S (revision 312198) @@ -1,29 +1,33 @@ //===-- truncdfsf2vfp.S - Implement truncdfsf2vfp -------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern float __truncdfsf2vfp(double a); // // Converts double precision float to signle precision result. // Uses Darwin calling convention where a double precision parameter is // passed in a R0/R1 pair and a signle precision result is returned in R0. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__truncdfsf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcvt.f32.f64 s0, d0 +#else vmov d7, r0, r1 // load double from r0/r1 pair vcvt.f32.f64 s15, d7 // convert double to single (trucate precision) vmov r0, s15 // return result in r0 +#endif bx lr END_COMPILERRT_FUNCTION(__truncdfsf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/unorddf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/unorddf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/unorddf2vfp.S (revision 312198) @@ -1,32 +1,36 @@ //===-- unorddf2vfp.S - Implement unorddf2vfp ------------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern int __unorddf2vfp(double a, double b); // // Returns one iff a or b is NaN // Uses Darwin calling convention where double precision arguments are passsed // like in GPR pairs. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__unorddf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f64 d0, d1 +#else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register - vcmp.f64 d6, d7 + vcmp.f64 d6, d7 +#endif vmrs apsr_nzcv, fpscr movvs r0, #1 // set result register to 1 if "overflow" (any NaNs) movvc r0, #0 bx lr END_COMPILERRT_FUNCTION(__unorddf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/unordsf2vfp.S =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/unordsf2vfp.S (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/builtins/arm/unordsf2vfp.S (revision 312198) @@ -1,32 +1,36 @@ //===-- unordsf2vfp.S - Implement unordsf2vfp -----------------------------===// // // The LLVM Compiler Infrastructure // // This file is dual licensed under the MIT and the University of Illinois Open // Source Licenses. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "../assembly.h" // // extern int __unordsf2vfp(float a, float b); // // Returns one iff a or b is NaN // Uses Darwin calling convention where single precision arguments are passsed // like 32-bit ints // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__unordsf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f32 s0, s1 +#else vmov s14, r0 // move from GPR 0 to float register vmov s15, r1 // move from GPR 1 to float register vcmp.f32 s14, s15 +#endif vmrs apsr_nzcv, fpscr movvs r0, #1 // set result register to 1 if "overflow" (any NaNs) movvc r0, #0 bx lr END_COMPILERRT_FUNCTION(__unordsf2vfp) NO_EXEC_STACK_DIRECTIVE Index: projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_common.h (revision 312198) @@ -1,919 +1,920 @@ //===-- sanitizer_common.h --------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file is shared between run-time libraries of sanitizers. // // It declares common functions and classes that are used in both runtimes. // Implementation of some functions are provided in sanitizer_common, while // others must be defined by run-time library itself. //===----------------------------------------------------------------------===// #ifndef SANITIZER_COMMON_H #define SANITIZER_COMMON_H #include "sanitizer_flags.h" #include "sanitizer_interface_internal.h" #include "sanitizer_internal_defs.h" #include "sanitizer_libc.h" #include "sanitizer_list.h" #include "sanitizer_mutex.h" #if defined(_MSC_VER) && !defined(__clang__) extern "C" void _ReadWriteBarrier(); #pragma intrinsic(_ReadWriteBarrier) #endif namespace __sanitizer { struct StackTrace; struct AddressInfo; // Constants. const uptr kWordSize = SANITIZER_WORDSIZE / 8; const uptr kWordSizeInBits = 8 * kWordSize; #if defined(__powerpc__) || defined(__powerpc64__) const uptr kCacheLineSize = 128; #else const uptr kCacheLineSize = 64; #endif const uptr kMaxPathLength = 4096; const uptr kMaxThreadStackSize = 1 << 30; // 1Gb static const uptr kErrorMessageBufferSize = 1 << 16; // Denotes fake PC values that come from JIT/JAVA/etc. // For such PC values __tsan_symbolize_external() will be called. const u64 kExternalPCBit = 1ULL << 60; extern const char *SanitizerToolName; // Can be changed by the tool. extern atomic_uint32_t current_verbosity; INLINE void SetVerbosity(int verbosity) { atomic_store(¤t_verbosity, verbosity, memory_order_relaxed); } INLINE int Verbosity() { return atomic_load(¤t_verbosity, memory_order_relaxed); } uptr GetPageSize(); extern uptr PageSizeCached; INLINE uptr GetPageSizeCached() { if (!PageSizeCached) PageSizeCached = GetPageSize(); return PageSizeCached; } uptr GetMmapGranularity(); uptr GetMaxVirtualAddress(); // Threads uptr GetTid(); uptr GetThreadSelf(); void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, uptr *stack_bottom); void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, uptr *tls_addr, uptr *tls_size); // Memory management void *MmapOrDie(uptr size, const char *mem_type, bool raw_report = false); INLINE void *MmapOrDieQuietly(uptr size, const char *mem_type) { return MmapOrDie(size, mem_type, /*raw_report*/ true); } void UnmapOrDie(void *addr, uptr size); void *MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name = nullptr); void *MmapNoReserveOrDie(uptr size, const char *mem_type); void *MmapFixedOrDie(uptr fixed_addr, uptr size); void *MmapFixedNoAccess(uptr fixed_addr, uptr size, const char *name = nullptr); void *MmapNoAccess(uptr size); // Map aligned chunk of address space; size and alignment are powers of two. void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type); // Disallow access to a memory range. Use MmapFixedNoAccess to allocate an // unaccessible memory. bool MprotectNoAccess(uptr addr, uptr size); bool MprotectReadOnly(uptr addr, uptr size); // Find an available address space. uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding); // Used to check if we can map shadow memory to a fixed location. bool MemoryRangeIsAvailable(uptr range_start, uptr range_end); // Releases memory pages entirely within the [beg, end] address range. Noop if // the provided range does not contain at least one entire page. void ReleaseMemoryPagesToOS(uptr beg, uptr end); void IncreaseTotalMmap(uptr size); void DecreaseTotalMmap(uptr size); uptr GetRSS(); void NoHugePagesInRegion(uptr addr, uptr length); void DontDumpShadowMemory(uptr addr, uptr length); // Check if the built VMA size matches the runtime one. void CheckVMASize(); void RunMallocHooks(const void *ptr, uptr size); void RunFreeHooks(const void *ptr); // InternalScopedBuffer can be used instead of large stack arrays to // keep frame size low. // FIXME: use InternalAlloc instead of MmapOrDie once // InternalAlloc is made libc-free. template class InternalScopedBuffer { public: explicit InternalScopedBuffer(uptr cnt) { cnt_ = cnt; ptr_ = (T *)MmapOrDie(cnt * sizeof(T), "InternalScopedBuffer"); } ~InternalScopedBuffer() { UnmapOrDie(ptr_, cnt_ * sizeof(T)); } T &operator[](uptr i) { return ptr_[i]; } T *data() { return ptr_; } uptr size() { return cnt_ * sizeof(T); } private: T *ptr_; uptr cnt_; // Disallow copies and moves. InternalScopedBuffer(const InternalScopedBuffer &) = delete; InternalScopedBuffer &operator=(const InternalScopedBuffer &) = delete; InternalScopedBuffer(InternalScopedBuffer &&) = delete; InternalScopedBuffer &operator=(InternalScopedBuffer &&) = delete; }; class InternalScopedString : public InternalScopedBuffer { public: explicit InternalScopedString(uptr max_length) : InternalScopedBuffer(max_length), length_(0) { (*this)[0] = '\0'; } uptr length() { return length_; } void clear() { (*this)[0] = '\0'; length_ = 0; } void append(const char *format, ...); private: uptr length_; }; // Simple low-level (mmap-based) allocator for internal use. Doesn't have // constructor, so all instances of LowLevelAllocator should be // linker initialized. class LowLevelAllocator { public: // Requires an external lock. void *Allocate(uptr size); private: char *allocated_end_; char *allocated_current_; }; typedef void (*LowLevelAllocateCallback)(uptr ptr, uptr size); // Allows to register tool-specific callbacks for LowLevelAllocator. // Passing NULL removes the callback. void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback); // IO void RawWrite(const char *buffer); bool ColorizeReports(); void RemoveANSIEscapeSequencesFromString(char *buffer); void Printf(const char *format, ...); void Report(const char *format, ...); void SetPrintfAndReportCallback(void (*callback)(const char *)); #define VReport(level, ...) \ do { \ if ((uptr)Verbosity() >= (level)) Report(__VA_ARGS__); \ } while (0) #define VPrintf(level, ...) \ do { \ if ((uptr)Verbosity() >= (level)) Printf(__VA_ARGS__); \ } while (0) // Can be used to prevent mixing error reports from different sanitizers. extern StaticSpinMutex CommonSanitizerReportMutex; struct ReportFile { void Write(const char *buffer, uptr length); bool SupportsColors(); void SetReportPath(const char *path); // Don't use fields directly. They are only declared public to allow // aggregate initialization. // Protects fields below. StaticSpinMutex *mu; // Opened file descriptor. Defaults to stderr. It may be equal to // kInvalidFd, in which case new file will be opened when necessary. fd_t fd; // Path prefix of report file, set via __sanitizer_set_report_path. char path_prefix[kMaxPathLength]; // Full path to report, obtained as .PID char full_path[kMaxPathLength]; // PID of the process that opened fd. If a fork() occurs, // the PID of child will be different from fd_pid. uptr fd_pid; private: void ReopenIfNecessary(); }; extern ReportFile report_file; extern uptr stoptheworld_tracer_pid; extern uptr stoptheworld_tracer_ppid; enum FileAccessMode { RdOnly, WrOnly, RdWr }; // Returns kInvalidFd on error. fd_t OpenFile(const char *filename, FileAccessMode mode, error_t *errno_p = nullptr); void CloseFile(fd_t); // Return true on success, false on error. bool ReadFromFile(fd_t fd, void *buff, uptr buff_size, uptr *bytes_read = nullptr, error_t *error_p = nullptr); bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, uptr *bytes_written = nullptr, error_t *error_p = nullptr); bool RenameFile(const char *oldpath, const char *newpath, error_t *error_p = nullptr); // Scoped file handle closer. struct FileCloser { explicit FileCloser(fd_t fd) : fd(fd) {} ~FileCloser() { CloseFile(fd); } fd_t fd; }; bool SupportsColoredOutput(fd_t fd); // Opens the file 'file_name" and reads up to 'max_len' bytes. // The resulting buffer is mmaped and stored in '*buff'. // The size of the mmaped region is stored in '*buff_size'. // The total number of read bytes is stored in '*read_len'. // Returns true if file was successfully opened and read. bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, uptr *read_len, uptr max_len = 1 << 26, error_t *errno_p = nullptr); // Maps given file to virtual memory, and returns pointer to it // (or NULL if mapping fails). Stores the size of mmaped region // in '*buff_size'. void *MapFileToMemory(const char *file_name, uptr *buff_size); void *MapWritableFileToMemory(void *addr, uptr size, fd_t fd, OFF_T offset); bool IsAccessibleMemoryRange(uptr beg, uptr size); // Error report formatting. const char *StripPathPrefix(const char *filepath, const char *strip_file_prefix); // Strip the directories from the module name. const char *StripModuleName(const char *module); // OS uptr ReadBinaryName(/*out*/char *buf, uptr buf_len); uptr ReadBinaryNameCached(/*out*/char *buf, uptr buf_len); uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len); const char *GetProcessName(); void UpdateProcessName(); void CacheBinaryName(); void DisableCoreDumperIfNecessary(); void DumpProcessMap(); void PrintModuleMap(); bool FileExists(const char *filename); const char *GetEnv(const char *name); bool SetEnv(const char *name, const char *value); const char *GetPwd(); char *FindPathToBinary(const char *name); bool IsPathSeparator(const char c); bool IsAbsolutePath(const char *path); // Starts a subprocess and returs its pid. // If *_fd parameters are not kInvalidFd their corresponding input/output // streams will be redirect to the file. The files will always be closed // in parent process even in case of an error. // The child process will close all fds after STDERR_FILENO // before passing control to a program. pid_t StartSubprocess(const char *filename, const char *const argv[], fd_t stdin_fd = kInvalidFd, fd_t stdout_fd = kInvalidFd, fd_t stderr_fd = kInvalidFd); // Checks if specified process is still running bool IsProcessRunning(pid_t pid); // Waits for the process to finish and returns its exit code. // Returns -1 in case of an error. int WaitForProcess(pid_t pid); u32 GetUid(); void ReExec(); char **GetArgv(); void PrintCmdline(); bool StackSizeIsUnlimited(); uptr GetStackSizeLimitInBytes(); void SetStackSizeLimitInBytes(uptr limit); bool AddressSpaceIsUnlimited(); void SetAddressSpaceUnlimited(); void AdjustStackSize(void *attr); void PrepareForSandboxing(__sanitizer_sandbox_arguments *args); void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args); void SetSandboxingCallback(void (*f)()); void CoverageUpdateMapping(); void CovBeforeFork(); void CovAfterFork(int child_pid); void InitializeCoverage(bool enabled, const char *coverage_dir); void ReInitializeCoverage(bool enabled, const char *coverage_dir); void InitTlsSize(); uptr GetTlsSize(); // Other void SleepForSeconds(int seconds); void SleepForMillis(int millis); u64 NanoTime(); int Atexit(void (*function)(void)); void SortArray(uptr *array, uptr size); void SortArray(u32 *array, uptr size); bool TemplateMatch(const char *templ, const char *str); // Exit void NORETURN Abort(); void NORETURN Die(); void NORETURN CheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2); void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type, const char *mmap_type, error_t err, bool raw_report = false); // Set the name of the current thread to 'name', return true on succees. // The name may be truncated to a system-dependent limit. bool SanitizerSetThreadName(const char *name); // Get the name of the current thread (no more than max_len bytes), // return true on succees. name should have space for at least max_len+1 bytes. bool SanitizerGetThreadName(char *name, int max_len); // Specific tools may override behavior of "Die" and "CheckFailed" functions // to do tool-specific job. typedef void (*DieCallbackType)(void); // It's possible to add several callbacks that would be run when "Die" is // called. The callbacks will be run in the opposite order. The tools are // strongly recommended to setup all callbacks during initialization, when there // is only a single thread. bool AddDieCallback(DieCallbackType callback); bool RemoveDieCallback(DieCallbackType callback); void SetUserDieCallback(DieCallbackType callback); typedef void (*CheckFailedCallbackType)(const char *, int, const char *, u64, u64); void SetCheckFailedCallback(CheckFailedCallbackType callback); // Callback will be called if soft_rss_limit_mb is given and the limit is // exceeded (exceeded==true) or if rss went down below the limit // (exceeded==false). // The callback should be registered once at the tool init time. void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)); // Functions related to signal handling. typedef void (*SignalHandlerType)(int, void *, void *); bool IsHandledDeadlySignal(int signum); void InstallDeadlySignalHandlers(SignalHandlerType handler); // Alternative signal stack (POSIX-only). void SetAlternateSignalStack(); void UnsetAlternateSignalStack(); // We don't want a summary too long. const int kMaxSummaryLength = 1024; // Construct a one-line string: // SUMMARY: SanitizerToolName: error_message // and pass it to __sanitizer_report_error_summary. void ReportErrorSummary(const char *error_message); // Same as above, but construct error_message as: // error_type file:line[:column][ function] void ReportErrorSummary(const char *error_type, const AddressInfo &info); // Same as above, but obtains AddressInfo by symbolizing top stack trace frame. void ReportErrorSummary(const char *error_type, const StackTrace *trace); // Math #if SANITIZER_WINDOWS && !defined(__clang__) && !defined(__GNUC__) extern "C" { unsigned char _BitScanForward(unsigned long *index, unsigned long mask); // NOLINT unsigned char _BitScanReverse(unsigned long *index, unsigned long mask); // NOLINT #if defined(_WIN64) unsigned char _BitScanForward64(unsigned long *index, unsigned __int64 mask); // NOLINT unsigned char _BitScanReverse64(unsigned long *index, unsigned __int64 mask); // NOLINT #endif } #endif INLINE uptr MostSignificantSetBitIndex(uptr x) { CHECK_NE(x, 0U); unsigned long up; // NOLINT #if !SANITIZER_WINDOWS || defined(__clang__) || defined(__GNUC__) # ifdef _WIN64 up = SANITIZER_WORDSIZE - 1 - __builtin_clzll(x); # else up = SANITIZER_WORDSIZE - 1 - __builtin_clzl(x); # endif #elif defined(_WIN64) _BitScanReverse64(&up, x); #else _BitScanReverse(&up, x); #endif return up; } INLINE uptr LeastSignificantSetBitIndex(uptr x) { CHECK_NE(x, 0U); unsigned long up; // NOLINT #if !SANITIZER_WINDOWS || defined(__clang__) || defined(__GNUC__) # ifdef _WIN64 up = __builtin_ctzll(x); # else up = __builtin_ctzl(x); # endif #elif defined(_WIN64) _BitScanForward64(&up, x); #else _BitScanForward(&up, x); #endif return up; } INLINE bool IsPowerOfTwo(uptr x) { return (x & (x - 1)) == 0; } INLINE uptr RoundUpToPowerOfTwo(uptr size) { CHECK(size); if (IsPowerOfTwo(size)) return size; uptr up = MostSignificantSetBitIndex(size); CHECK_LT(size, (1ULL << (up + 1))); CHECK_GT(size, (1ULL << up)); return 1ULL << (up + 1); } INLINE uptr RoundUpTo(uptr size, uptr boundary) { RAW_CHECK(IsPowerOfTwo(boundary)); return (size + boundary - 1) & ~(boundary - 1); } INLINE uptr RoundDownTo(uptr x, uptr boundary) { return x & ~(boundary - 1); } INLINE bool IsAligned(uptr a, uptr alignment) { return (a & (alignment - 1)) == 0; } INLINE uptr Log2(uptr x) { CHECK(IsPowerOfTwo(x)); return LeastSignificantSetBitIndex(x); } // Don't use std::min, std::max or std::swap, to minimize dependency // on libstdc++. template T Min(T a, T b) { return a < b ? a : b; } template T Max(T a, T b) { return a > b ? a : b; } template void Swap(T& a, T& b) { T tmp = a; a = b; b = tmp; } // Char handling INLINE bool IsSpace(int c) { return (c == ' ') || (c == '\n') || (c == '\t') || (c == '\f') || (c == '\r') || (c == '\v'); } INLINE bool IsDigit(int c) { return (c >= '0') && (c <= '9'); } INLINE int ToLower(int c) { return (c >= 'A' && c <= 'Z') ? (c + 'a' - 'A') : c; } // A low-level vector based on mmap. May incur a significant memory overhead for // small vectors. // WARNING: The current implementation supports only POD types. template class InternalMmapVectorNoCtor { public: void Initialize(uptr initial_capacity) { capacity_ = Max(initial_capacity, (uptr)1); size_ = 0; data_ = (T *)MmapOrDie(capacity_ * sizeof(T), "InternalMmapVectorNoCtor"); } void Destroy() { UnmapOrDie(data_, capacity_ * sizeof(T)); } T &operator[](uptr i) { CHECK_LT(i, size_); return data_[i]; } const T &operator[](uptr i) const { CHECK_LT(i, size_); return data_[i]; } void push_back(const T &element) { CHECK_LE(size_, capacity_); if (size_ == capacity_) { uptr new_capacity = RoundUpToPowerOfTwo(size_ + 1); Resize(new_capacity); } internal_memcpy(&data_[size_++], &element, sizeof(T)); } T &back() { CHECK_GT(size_, 0); return data_[size_ - 1]; } void pop_back() { CHECK_GT(size_, 0); size_--; } uptr size() const { return size_; } const T *data() const { return data_; } T *data() { return data_; } uptr capacity() const { return capacity_; } void resize(uptr new_size) { Resize(new_size); if (new_size > size_) { internal_memset(&data_[size_], 0, sizeof(T) * (new_size - size_)); } size_ = new_size; } void clear() { size_ = 0; } bool empty() const { return size() == 0; } const T *begin() const { return data(); } T *begin() { return data(); } const T *end() const { return data() + size(); } T *end() { return data() + size(); } private: void Resize(uptr new_capacity) { CHECK_GT(new_capacity, 0); CHECK_LE(size_, new_capacity); T *new_data = (T *)MmapOrDie(new_capacity * sizeof(T), "InternalMmapVector"); internal_memcpy(new_data, data_, size_ * sizeof(T)); T *old_data = data_; data_ = new_data; UnmapOrDie(old_data, capacity_ * sizeof(T)); capacity_ = new_capacity; } T *data_; uptr capacity_; uptr size_; }; template class InternalMmapVector : public InternalMmapVectorNoCtor { public: explicit InternalMmapVector(uptr initial_capacity) { InternalMmapVectorNoCtor::Initialize(initial_capacity); } ~InternalMmapVector() { InternalMmapVectorNoCtor::Destroy(); } // Disallow evil constructors. InternalMmapVector(const InternalMmapVector&); void operator=(const InternalMmapVector&); }; // HeapSort for arrays and InternalMmapVector. template void InternalSort(Container *v, uptr size, Compare comp) { if (size < 2) return; // Stage 1: insert elements to the heap. for (uptr i = 1; i < size; i++) { uptr j, p; for (j = i; j > 0; j = p) { p = (j - 1) / 2; if (comp((*v)[p], (*v)[j])) Swap((*v)[j], (*v)[p]); else break; } } // Stage 2: swap largest element with the last one, // and sink the new top. for (uptr i = size - 1; i > 0; i--) { Swap((*v)[0], (*v)[i]); uptr j, max_ind; for (j = 0; j < i; j = max_ind) { uptr left = 2 * j + 1; uptr right = 2 * j + 2; max_ind = j; if (left < i && comp((*v)[max_ind], (*v)[left])) max_ind = left; if (right < i && comp((*v)[max_ind], (*v)[right])) max_ind = right; if (max_ind != j) Swap((*v)[j], (*v)[max_ind]); else break; } } } // Works like std::lower_bound: finds the first element that is not less // than the val. template uptr InternalLowerBound(const Container &v, uptr first, uptr last, const Value &val, Compare comp) { while (last > first) { uptr mid = (first + last) / 2; if (comp(v[mid], val)) first = mid + 1; else last = mid; } return first; } enum ModuleArch { kModuleArchUnknown, kModuleArchI386, kModuleArchX86_64, kModuleArchX86_64H, kModuleArchARMV6, kModuleArchARMV7, kModuleArchARMV7S, kModuleArchARMV7K, kModuleArchARM64 }; // When adding a new architecture, don't forget to also update // script/asan_symbolize.py and sanitizer_symbolizer_libcdep.cc. inline const char *ModuleArchToString(ModuleArch arch) { switch (arch) { case kModuleArchUnknown: return ""; case kModuleArchI386: return "i386"; case kModuleArchX86_64: return "x86_64"; case kModuleArchX86_64H: return "x86_64h"; case kModuleArchARMV6: return "armv6"; case kModuleArchARMV7: return "armv7"; case kModuleArchARMV7S: return "armv7s"; case kModuleArchARMV7K: return "armv7k"; case kModuleArchARM64: return "arm64"; } CHECK(0 && "Invalid module arch"); + return ""; } const uptr kModuleUUIDSize = 16; // Represents a binary loaded into virtual memory (e.g. this can be an // executable or a shared object). class LoadedModule { public: LoadedModule() : full_name_(nullptr), base_address_(0), max_executable_address_(0), arch_(kModuleArchUnknown), instrumented_(false) { internal_memset(uuid_, 0, kModuleUUIDSize); ranges_.clear(); } void set(const char *module_name, uptr base_address); void set(const char *module_name, uptr base_address, ModuleArch arch, u8 uuid[kModuleUUIDSize], bool instrumented); void clear(); void addAddressRange(uptr beg, uptr end, bool executable); bool containsAddress(uptr address) const; const char *full_name() const { return full_name_; } uptr base_address() const { return base_address_; } uptr max_executable_address() const { return max_executable_address_; } ModuleArch arch() const { return arch_; } const u8 *uuid() const { return uuid_; } bool instrumented() const { return instrumented_; } struct AddressRange { AddressRange *next; uptr beg; uptr end; bool executable; AddressRange(uptr beg, uptr end, bool executable) : next(nullptr), beg(beg), end(end), executable(executable) {} }; const IntrusiveList &ranges() const { return ranges_; } private: char *full_name_; // Owned. uptr base_address_; uptr max_executable_address_; ModuleArch arch_; u8 uuid_[kModuleUUIDSize]; bool instrumented_; IntrusiveList ranges_; }; // List of LoadedModules. OS-dependent implementation is responsible for // filling this information. class ListOfModules { public: ListOfModules() : modules_(kInitialCapacity) {} ~ListOfModules() { clear(); } void init(); const LoadedModule *begin() const { return modules_.begin(); } LoadedModule *begin() { return modules_.begin(); } const LoadedModule *end() const { return modules_.end(); } LoadedModule *end() { return modules_.end(); } uptr size() const { return modules_.size(); } const LoadedModule &operator[](uptr i) const { CHECK_LT(i, modules_.size()); return modules_[i]; } private: void clear() { for (auto &module : modules_) module.clear(); modules_.clear(); } InternalMmapVector modules_; // We rarely have more than 16K loaded modules. static const uptr kInitialCapacity = 1 << 14; }; // Callback type for iterating over a set of memory ranges. typedef void (*RangeIteratorCallback)(uptr begin, uptr end, void *arg); enum AndroidApiLevel { ANDROID_NOT_ANDROID = 0, ANDROID_KITKAT = 19, ANDROID_LOLLIPOP_MR1 = 22, ANDROID_POST_LOLLIPOP = 23 }; void WriteToSyslog(const char *buffer); #if SANITIZER_MAC void LogFullErrorReport(const char *buffer); #else INLINE void LogFullErrorReport(const char *buffer) {} #endif #if SANITIZER_LINUX || SANITIZER_MAC void WriteOneLineToSyslog(const char *s); void LogMessageOnPrintf(const char *str); #else INLINE void WriteOneLineToSyslog(const char *s) {} INLINE void LogMessageOnPrintf(const char *str) {} #endif #if SANITIZER_LINUX // Initialize Android logging. Any writes before this are silently lost. void AndroidLogInit(); #else INLINE void AndroidLogInit() {} #endif #if SANITIZER_ANDROID void SanitizerInitializeUnwinder(); AndroidApiLevel AndroidGetApiLevel(); #else INLINE void AndroidLogWrite(const char *buffer_unused) {} INLINE void SanitizerInitializeUnwinder() {} INLINE AndroidApiLevel AndroidGetApiLevel() { return ANDROID_NOT_ANDROID; } #endif INLINE uptr GetPthreadDestructorIterations() { #if SANITIZER_ANDROID return (AndroidGetApiLevel() == ANDROID_LOLLIPOP_MR1) ? 8 : 4; #elif SANITIZER_POSIX return 4; #else // Unused on Windows. return 0; #endif } void *internal_start_thread(void(*func)(void*), void *arg); void internal_join_thread(void *th); void MaybeStartBackgroudThread(); // Make the compiler think that something is going on there. // Use this inside a loop that looks like memset/memcpy/etc to prevent the // compiler from recognising it and turning it into an actual call to // memset/memcpy/etc. static inline void SanitizerBreakOptimization(void *arg) { #if defined(_MSC_VER) && !defined(__clang__) _ReadWriteBarrier(); #else __asm__ __volatile__("" : : "r" (arg) : "memory"); #endif } struct SignalContext { void *context; uptr addr; uptr pc; uptr sp; uptr bp; bool is_memory_access; enum WriteFlag { UNKNOWN, READ, WRITE } write_flag; SignalContext(void *context, uptr addr, uptr pc, uptr sp, uptr bp, bool is_memory_access, WriteFlag write_flag) : context(context), addr(addr), pc(pc), sp(sp), bp(bp), is_memory_access(is_memory_access), write_flag(write_flag) {} static void DumpAllRegisters(void *context); // Creates signal context in a platform-specific manner. static SignalContext Create(void *siginfo, void *context); // Returns true if the "context" indicates a memory write. static WriteFlag GetWriteFlag(void *context); }; void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp); void MaybeReexec(); template class RunOnDestruction { public: explicit RunOnDestruction(Fn fn) : fn_(fn) {} ~RunOnDestruction() { fn_(); } private: Fn fn_; }; // A simple scope guard. Usage: // auto cleanup = at_scope_exit([]{ do_cleanup; }); template RunOnDestruction at_scope_exit(Fn fn) { return RunOnDestruction(fn); } // Linux on 64-bit s390 had a nasty bug that crashes the whole machine // if a process uses virtual memory over 4TB (as many sanitizers like // to do). This function will abort the process if running on a kernel // that looks vulnerable. #if SANITIZER_LINUX && SANITIZER_S390_64 void AvoidCVE_2016_2143(); #else INLINE void AvoidCVE_2016_2143() {} #endif struct StackDepotStats { uptr n_uniq_ids; uptr allocated; }; // The default value for allocator_release_to_os_interval_ms common flag to // indicate that sanitizer allocator should not attempt to release memory to OS. const s32 kReleaseToOSIntervalNever = -1; } // namespace __sanitizer inline void *operator new(__sanitizer::operator_new_size_type size, __sanitizer::LowLevelAllocator &alloc) { return alloc.Allocate(size); } #endif // SANITIZER_COMMON_H Index: projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep.cc (revision 312198) @@ -1,1042 +1,1044 @@ //===-- sanitizer_coverage.cc ---------------------------------------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // Sanitizer Coverage. // This file implements run-time support for a poor man's coverage tool. // // Compiler instrumentation: // For every interesting basic block the compiler injects the following code: // if (Guard < 0) { // __sanitizer_cov(&Guard); // } // At the module start up time __sanitizer_cov_module_init sets the guards // to consecutive negative numbers (-1, -2, -3, ...). // It's fine to call __sanitizer_cov more than once for a given block. // // Run-time: // - __sanitizer_cov(): record that we've executed the PC (GET_CALLER_PC). // and atomically set Guard to -Guard. // - __sanitizer_cov_dump: dump the coverage data to disk. // For every module of the current process that has coverage data // this will create a file module_name.PID.sancov. // // The file format is simple: the first 8 bytes is the magic, // one of 0xC0BFFFFFFFFFFF64 and 0xC0BFFFFFFFFFFF32. The last byte of the // magic defines the size of the following offsets. // The rest of the data is the offsets in the module. // // Eventually, this coverage implementation should be obsoleted by a more // powerful general purpose Clang/LLVM coverage instrumentation. // Consider this implementation as prototype. // // FIXME: support (or at least test with) dlclose. //===----------------------------------------------------------------------===// #include "sanitizer_allocator_internal.h" #include "sanitizer_common.h" #include "sanitizer_libc.h" #include "sanitizer_mutex.h" #include "sanitizer_procmaps.h" #include "sanitizer_stacktrace.h" #include "sanitizer_symbolizer.h" #include "sanitizer_flags.h" using namespace __sanitizer; static const u64 kMagic64 = 0xC0BFFFFFFFFFFF64ULL; static const u64 kMagic32 = 0xC0BFFFFFFFFFFF32ULL; static const uptr kNumWordsForMagic = SANITIZER_WORDSIZE == 64 ? 1 : 2; static const u64 kMagic = SANITIZER_WORDSIZE == 64 ? kMagic64 : kMagic32; static atomic_uint32_t dump_once_guard; // Ensure that CovDump runs only once. static atomic_uintptr_t coverage_counter; static atomic_uintptr_t caller_callee_counter; static void ResetGlobalCounters() { return atomic_store(&coverage_counter, 0, memory_order_relaxed); return atomic_store(&caller_callee_counter, 0, memory_order_relaxed); } // pc_array is the array containing the covered PCs. // To make the pc_array thread- and async-signal-safe it has to be large enough. // 128M counters "ought to be enough for anybody" (4M on 32-bit). // With coverage_direct=1 in ASAN_OPTIONS, pc_array memory is mapped to a file. // In this mode, __sanitizer_cov_dump does nothing, and CovUpdateMapping() // dump current memory layout to another file. static bool cov_sandboxed = false; static fd_t cov_fd = kInvalidFd; static unsigned int cov_max_block_size = 0; static bool coverage_enabled = false; static const char *coverage_dir; namespace __sanitizer { class CoverageData { public: void Init(); void Enable(); void Disable(); void ReInit(); void BeforeFork(); void AfterFork(int child_pid); void Extend(uptr npcs); void Add(uptr pc, u32 *guard); void IndirCall(uptr caller, uptr callee, uptr callee_cache[], uptr cache_size); void DumpCallerCalleePairs(); void DumpTrace(); void DumpAsBitSet(); void DumpCounters(); void DumpOffsets(); void DumpAll(); ALWAYS_INLINE void TraceBasicBlock(u32 *id); void InitializeGuardArray(s32 *guards); void InitializeGuards(s32 *guards, uptr n, const char *module_name, uptr caller_pc); void InitializeCounters(u8 *counters, uptr n); void ReinitializeGuards(); uptr GetNumberOf8bitCounters(); uptr Update8bitCounterBitsetAndClearCounters(u8 *bitset); uptr *data(); uptr size() const; private: struct NamedPcRange { const char *copied_module_name; uptr beg, end; // elements [beg,end) in pc_array. }; void DirectOpen(); void UpdateModuleNameVec(uptr caller_pc, uptr range_beg, uptr range_end); void GetRangeOffsets(const NamedPcRange& r, Symbolizer* s, InternalMmapVector* offsets) const; // Maximal size pc array may ever grow. // We MmapNoReserve this space to ensure that the array is contiguous. static const uptr kPcArrayMaxSize = FIRST_32_SECOND_64(1 << (SANITIZER_ANDROID ? 24 : 26), 1 << 27); // The amount file mapping for the pc array is grown by. static const uptr kPcArrayMmapSize = 64 * 1024; // pc_array is allocated with MmapNoReserveOrDie and so it uses only as // much RAM as it really needs. uptr *pc_array; // Index of the first available pc_array slot. atomic_uintptr_t pc_array_index; // Array size. atomic_uintptr_t pc_array_size; // Current file mapped size of the pc array. uptr pc_array_mapped_size; // Descriptor of the file mapped pc array. fd_t pc_fd; // Vector of coverage guard arrays, protected by mu. InternalMmapVectorNoCtor guard_array_vec; // Vector of module and compilation unit pc ranges. InternalMmapVectorNoCtor comp_unit_name_vec; InternalMmapVectorNoCtor module_name_vec; struct CounterAndSize { u8 *counters; uptr n; }; InternalMmapVectorNoCtor counters_vec; uptr num_8bit_counters; // Caller-Callee (cc) array, size and current index. static const uptr kCcArrayMaxSize = FIRST_32_SECOND_64(1 << 18, 1 << 24); uptr **cc_array; atomic_uintptr_t cc_array_index; atomic_uintptr_t cc_array_size; // Tracing event array, size and current pointer. // We record all events (basic block entries) in a global buffer of u32 // values. Each such value is the index in pc_array. // So far the tracing is highly experimental: // - not thread-safe; // - does not support long traces; // - not tuned for performance. static const uptr kTrEventArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 30); u32 *tr_event_array; uptr tr_event_array_size; u32 *tr_event_pointer; static const uptr kTrPcArrayMaxSize = FIRST_32_SECOND_64(1 << 22, 1 << 27); StaticSpinMutex mu; }; static CoverageData coverage_data; void CovUpdateMapping(const char *path, uptr caller_pc = 0); void CoverageData::DirectOpen() { InternalScopedString path(kMaxPathLength); internal_snprintf((char *)path.data(), path.size(), "%s/%zd.sancov.raw", coverage_dir, internal_getpid()); pc_fd = OpenFile(path.data(), RdWr); if (pc_fd == kInvalidFd) { Report("Coverage: failed to open %s for reading/writing\n", path.data()); Die(); } pc_array_mapped_size = 0; CovUpdateMapping(coverage_dir); } void CoverageData::Init() { pc_fd = kInvalidFd; } void CoverageData::Enable() { if (pc_array) return; pc_array = reinterpret_cast( MmapNoReserveOrDie(sizeof(uptr) * kPcArrayMaxSize, "CovInit")); atomic_store(&pc_array_index, 0, memory_order_relaxed); if (common_flags()->coverage_direct) { atomic_store(&pc_array_size, 0, memory_order_relaxed); } else { atomic_store(&pc_array_size, kPcArrayMaxSize, memory_order_relaxed); } cc_array = reinterpret_cast(MmapNoReserveOrDie( sizeof(uptr *) * kCcArrayMaxSize, "CovInit::cc_array")); atomic_store(&cc_array_size, kCcArrayMaxSize, memory_order_relaxed); atomic_store(&cc_array_index, 0, memory_order_relaxed); // Allocate tr_event_array with a guard page at the end. tr_event_array = reinterpret_cast(MmapNoReserveOrDie( sizeof(tr_event_array[0]) * kTrEventArrayMaxSize + GetMmapGranularity(), "CovInit::tr_event_array")); MprotectNoAccess( reinterpret_cast(&tr_event_array[kTrEventArrayMaxSize]), GetMmapGranularity()); tr_event_array_size = kTrEventArrayMaxSize; tr_event_pointer = tr_event_array; num_8bit_counters = 0; } void CoverageData::InitializeGuardArray(s32 *guards) { Enable(); // Make sure coverage is enabled at this point. s32 n = guards[0]; for (s32 j = 1; j <= n; j++) { uptr idx = atomic_load_relaxed(&pc_array_index); atomic_store_relaxed(&pc_array_index, idx + 1); guards[j] = -static_cast(idx + 1); } } void CoverageData::Disable() { if (pc_array) { UnmapOrDie(pc_array, sizeof(uptr) * kPcArrayMaxSize); pc_array = nullptr; } if (cc_array) { UnmapOrDie(cc_array, sizeof(uptr *) * kCcArrayMaxSize); cc_array = nullptr; } if (tr_event_array) { UnmapOrDie(tr_event_array, sizeof(tr_event_array[0]) * kTrEventArrayMaxSize + GetMmapGranularity()); tr_event_array = nullptr; tr_event_pointer = nullptr; } if (pc_fd != kInvalidFd) { CloseFile(pc_fd); pc_fd = kInvalidFd; } } void CoverageData::ReinitializeGuards() { // Assuming single thread. atomic_store(&pc_array_index, 0, memory_order_relaxed); for (uptr i = 0; i < guard_array_vec.size(); i++) InitializeGuardArray(guard_array_vec[i]); } void CoverageData::ReInit() { Disable(); if (coverage_enabled) { if (common_flags()->coverage_direct) { // In memory-mapped mode we must extend the new file to the known array // size. uptr size = atomic_load(&pc_array_size, memory_order_relaxed); uptr npcs = size / sizeof(uptr); Enable(); if (size) Extend(npcs); if (coverage_enabled) CovUpdateMapping(coverage_dir); } else { Enable(); } } // Re-initialize the guards. // We are single-threaded now, no need to grab any lock. CHECK_EQ(atomic_load(&pc_array_index, memory_order_relaxed), 0); ReinitializeGuards(); } void CoverageData::BeforeFork() { mu.Lock(); } void CoverageData::AfterFork(int child_pid) { // We are single-threaded so it's OK to release the lock early. mu.Unlock(); if (child_pid == 0) ReInit(); } // Extend coverage PC array to fit additional npcs elements. void CoverageData::Extend(uptr npcs) { if (!common_flags()->coverage_direct) return; SpinMutexLock l(&mu); uptr size = atomic_load(&pc_array_size, memory_order_relaxed); size += npcs * sizeof(uptr); if (coverage_enabled && size > pc_array_mapped_size) { if (pc_fd == kInvalidFd) DirectOpen(); CHECK_NE(pc_fd, kInvalidFd); uptr new_mapped_size = pc_array_mapped_size; while (size > new_mapped_size) new_mapped_size += kPcArrayMmapSize; CHECK_LE(new_mapped_size, sizeof(uptr) * kPcArrayMaxSize); // Extend the file and map the new space at the end of pc_array. uptr res = internal_ftruncate(pc_fd, new_mapped_size); int err; if (internal_iserror(res, &err)) { Printf("failed to extend raw coverage file: %d\n", err); Die(); } uptr next_map_base = ((uptr)pc_array) + pc_array_mapped_size; void *p = MapWritableFileToMemory((void *)next_map_base, new_mapped_size - pc_array_mapped_size, pc_fd, pc_array_mapped_size); CHECK_EQ((uptr)p, next_map_base); pc_array_mapped_size = new_mapped_size; } atomic_store(&pc_array_size, size, memory_order_release); } void CoverageData::InitializeCounters(u8 *counters, uptr n) { if (!counters) return; CHECK_EQ(reinterpret_cast(counters) % 16, 0); n = RoundUpTo(n, 16); // The compiler must ensure that counters is 16-aligned. SpinMutexLock l(&mu); counters_vec.push_back({counters, n}); num_8bit_counters += n; } void CoverageData::UpdateModuleNameVec(uptr caller_pc, uptr range_beg, uptr range_end) { auto sym = Symbolizer::GetOrInit(); if (!sym) return; const char *module_name = sym->GetModuleNameForPc(caller_pc); if (!module_name) return; if (module_name_vec.empty() || module_name_vec.back().copied_module_name != module_name) module_name_vec.push_back({module_name, range_beg, range_end}); else module_name_vec.back().end = range_end; } void CoverageData::InitializeGuards(s32 *guards, uptr n, const char *comp_unit_name, uptr caller_pc) { // The array 'guards' has n+1 elements, we use the element zero // to store 'n'. CHECK_LT(n, 1 << 30); guards[0] = static_cast(n); InitializeGuardArray(guards); SpinMutexLock l(&mu); uptr range_end = atomic_load(&pc_array_index, memory_order_relaxed); uptr range_beg = range_end - n; comp_unit_name_vec.push_back({comp_unit_name, range_beg, range_end}); guard_array_vec.push_back(guards); UpdateModuleNameVec(caller_pc, range_beg, range_end); } static const uptr kBundleCounterBits = 16; // When coverage_order_pcs==true and SANITIZER_WORDSIZE==64 // we insert the global counter into the first 16 bits of the PC. uptr BundlePcAndCounter(uptr pc, uptr counter) { if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs) return pc; static const uptr kMaxCounter = (1 << kBundleCounterBits) - 1; if (counter > kMaxCounter) counter = kMaxCounter; CHECK_EQ(0, pc >> (SANITIZER_WORDSIZE - kBundleCounterBits)); return pc | (counter << (SANITIZER_WORDSIZE - kBundleCounterBits)); } uptr UnbundlePc(uptr bundle) { if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs) return bundle; return (bundle << kBundleCounterBits) >> kBundleCounterBits; } uptr UnbundleCounter(uptr bundle) { if (SANITIZER_WORDSIZE != 64 || !common_flags()->coverage_order_pcs) return 0; return bundle >> (SANITIZER_WORDSIZE - kBundleCounterBits); } // If guard is negative, atomically set it to -guard and store the PC in // pc_array. void CoverageData::Add(uptr pc, u32 *guard) { atomic_uint32_t *atomic_guard = reinterpret_cast(guard); s32 guard_value = atomic_load(atomic_guard, memory_order_relaxed); if (guard_value >= 0) return; atomic_store(atomic_guard, -guard_value, memory_order_relaxed); if (!pc_array) return; uptr idx = -guard_value - 1; if (idx >= atomic_load(&pc_array_index, memory_order_acquire)) return; // May happen after fork when pc_array_index becomes 0. CHECK_LT(idx * sizeof(uptr), atomic_load(&pc_array_size, memory_order_acquire)); uptr counter = atomic_fetch_add(&coverage_counter, 1, memory_order_relaxed); pc_array[idx] = BundlePcAndCounter(pc, counter); } // Registers a pair caller=>callee. // When a given caller is seen for the first time, the callee_cache is added // to the global array cc_array, callee_cache[0] is set to caller and // callee_cache[1] is set to cache_size. // Then we are trying to add callee to callee_cache [2,cache_size) if it is // not there yet. // If the cache is full we drop the callee (may want to fix this later). void CoverageData::IndirCall(uptr caller, uptr callee, uptr callee_cache[], uptr cache_size) { if (!cc_array) return; atomic_uintptr_t *atomic_callee_cache = reinterpret_cast(callee_cache); uptr zero = 0; if (atomic_compare_exchange_strong(&atomic_callee_cache[0], &zero, caller, memory_order_seq_cst)) { uptr idx = atomic_fetch_add(&cc_array_index, 1, memory_order_relaxed); CHECK_LT(idx * sizeof(uptr), atomic_load(&cc_array_size, memory_order_acquire)); callee_cache[1] = cache_size; cc_array[idx] = callee_cache; } CHECK_EQ(atomic_load(&atomic_callee_cache[0], memory_order_relaxed), caller); for (uptr i = 2; i < cache_size; i++) { uptr was = 0; if (atomic_compare_exchange_strong(&atomic_callee_cache[i], &was, callee, memory_order_seq_cst)) { atomic_fetch_add(&caller_callee_counter, 1, memory_order_relaxed); return; } if (was == callee) // Already have this callee. return; } } uptr CoverageData::GetNumberOf8bitCounters() { return num_8bit_counters; } // Map every 8bit counter to a 8-bit bitset and clear the counter. uptr CoverageData::Update8bitCounterBitsetAndClearCounters(u8 *bitset) { uptr num_new_bits = 0; uptr cur = 0; // For better speed we map 8 counters to 8 bytes of bitset at once. static const uptr kBatchSize = 8; CHECK_EQ(reinterpret_cast(bitset) % kBatchSize, 0); for (uptr i = 0, len = counters_vec.size(); i < len; i++) { u8 *c = counters_vec[i].counters; uptr n = counters_vec[i].n; CHECK_EQ(n % 16, 0); CHECK_EQ(cur % kBatchSize, 0); CHECK_EQ(reinterpret_cast(c) % kBatchSize, 0); if (!bitset) { internal_bzero_aligned16(c, n); cur += n; continue; } for (uptr j = 0; j < n; j += kBatchSize, cur += kBatchSize) { CHECK_LT(cur, num_8bit_counters); u64 *pc64 = reinterpret_cast(c + j); u64 *pb64 = reinterpret_cast(bitset + cur); u64 c64 = *pc64; u64 old_bits_64 = *pb64; u64 new_bits_64 = old_bits_64; if (c64) { *pc64 = 0; for (uptr k = 0; k < kBatchSize; k++) { u64 x = (c64 >> (8 * k)) & 0xff; if (x) { u64 bit = 0; /**/ if (x >= 128) bit = 128; else if (x >= 32) bit = 64; else if (x >= 16) bit = 32; else if (x >= 8) bit = 16; else if (x >= 4) bit = 8; else if (x >= 3) bit = 4; else if (x >= 2) bit = 2; else if (x >= 1) bit = 1; u64 mask = bit << (8 * k); if (!(new_bits_64 & mask)) { num_new_bits++; new_bits_64 |= mask; } } } *pb64 = new_bits_64; } } } CHECK_EQ(cur, num_8bit_counters); return num_new_bits; } uptr *CoverageData::data() { return pc_array; } uptr CoverageData::size() const { return atomic_load(&pc_array_index, memory_order_relaxed); } // Block layout for packed file format: header, followed by module name (no // trailing zero), followed by data blob. struct CovHeader { int pid; unsigned int module_name_length; unsigned int data_length; }; static void CovWritePacked(int pid, const char *module, const void *blob, unsigned int blob_size) { if (cov_fd == kInvalidFd) return; unsigned module_name_length = internal_strlen(module); CovHeader header = {pid, module_name_length, blob_size}; if (cov_max_block_size == 0) { // Writing to a file. Just go ahead. WriteToFile(cov_fd, &header, sizeof(header)); WriteToFile(cov_fd, module, module_name_length); WriteToFile(cov_fd, blob, blob_size); } else { // Writing to a socket. We want to split the data into appropriately sized // blocks. InternalScopedBuffer block(cov_max_block_size); CHECK_EQ((uptr)block.data(), (uptr)(CovHeader *)block.data()); uptr header_size_with_module = sizeof(header) + module_name_length; CHECK_LT(header_size_with_module, cov_max_block_size); unsigned int max_payload_size = cov_max_block_size - header_size_with_module; char *block_pos = block.data(); internal_memcpy(block_pos, &header, sizeof(header)); block_pos += sizeof(header); internal_memcpy(block_pos, module, module_name_length); block_pos += module_name_length; char *block_data_begin = block_pos; const char *blob_pos = (const char *)blob; while (blob_size > 0) { unsigned int payload_size = Min(blob_size, max_payload_size); blob_size -= payload_size; internal_memcpy(block_data_begin, blob_pos, payload_size); blob_pos += payload_size; ((CovHeader *)block.data())->data_length = payload_size; WriteToFile(cov_fd, block.data(), header_size_with_module + payload_size); } } } // If packed = false: .. (name = module name). // If packed = true and name == 0: ... // If packed = true and name != 0: .. (name is // user-supplied). static fd_t CovOpenFile(InternalScopedString *path, bool packed, const char *name, const char *extension = "sancov") { path->clear(); if (!packed) { CHECK(name); path->append("%s/%s.%zd.%s", coverage_dir, name, internal_getpid(), extension); } else { if (!name) path->append("%s/%zd.%s.packed", coverage_dir, internal_getpid(), extension); else path->append("%s/%s.%s.packed", coverage_dir, name, extension); } error_t err; fd_t fd = OpenFile(path->data(), WrOnly, &err); if (fd == kInvalidFd) Report("SanitizerCoverage: failed to open %s for writing (reason: %d)\n", path->data(), err); return fd; } // Dump trace PCs and trace events into two separate files. void CoverageData::DumpTrace() { uptr max_idx = tr_event_pointer - tr_event_array; if (!max_idx) return; auto sym = Symbolizer::GetOrInit(); if (!sym) return; InternalScopedString out(32 << 20); for (uptr i = 0, n = size(); i < n; i++) { const char *module_name = ""; uptr module_address = 0; sym->GetModuleNameAndOffsetForPC(UnbundlePc(pc_array[i]), &module_name, &module_address); out.append("%s 0x%zx\n", module_name, module_address); } InternalScopedString path(kMaxPathLength); fd_t fd = CovOpenFile(&path, false, "trace-points"); if (fd == kInvalidFd) return; WriteToFile(fd, out.data(), out.length()); CloseFile(fd); fd = CovOpenFile(&path, false, "trace-compunits"); if (fd == kInvalidFd) return; out.clear(); for (uptr i = 0; i < comp_unit_name_vec.size(); i++) out.append("%s\n", comp_unit_name_vec[i].copied_module_name); WriteToFile(fd, out.data(), out.length()); CloseFile(fd); fd = CovOpenFile(&path, false, "trace-events"); if (fd == kInvalidFd) return; uptr bytes_to_write = max_idx * sizeof(tr_event_array[0]); u8 *event_bytes = reinterpret_cast(tr_event_array); // The trace file could be huge, and may not be written with a single syscall. while (bytes_to_write) { uptr actually_written; if (WriteToFile(fd, event_bytes, bytes_to_write, &actually_written) && actually_written <= bytes_to_write) { bytes_to_write -= actually_written; event_bytes += actually_written; } else { break; } } CloseFile(fd); VReport(1, " CovDump: Trace: %zd PCs written\n", size()); VReport(1, " CovDump: Trace: %zd Events written\n", max_idx); } // This function dumps the caller=>callee pairs into a file as a sequence of // lines like "module_name offset". void CoverageData::DumpCallerCalleePairs() { uptr max_idx = atomic_load(&cc_array_index, memory_order_relaxed); if (!max_idx) return; auto sym = Symbolizer::GetOrInit(); if (!sym) return; InternalScopedString out(32 << 20); uptr total = 0; for (uptr i = 0; i < max_idx; i++) { uptr *cc_cache = cc_array[i]; CHECK(cc_cache); uptr caller = cc_cache[0]; uptr n_callees = cc_cache[1]; const char *caller_module_name = ""; uptr caller_module_address = 0; sym->GetModuleNameAndOffsetForPC(caller, &caller_module_name, &caller_module_address); for (uptr j = 2; j < n_callees; j++) { uptr callee = cc_cache[j]; if (!callee) break; total++; const char *callee_module_name = ""; uptr callee_module_address = 0; sym->GetModuleNameAndOffsetForPC(callee, &callee_module_name, &callee_module_address); out.append("%s 0x%zx\n%s 0x%zx\n", caller_module_name, caller_module_address, callee_module_name, callee_module_address); } } InternalScopedString path(kMaxPathLength); fd_t fd = CovOpenFile(&path, false, "caller-callee"); if (fd == kInvalidFd) return; WriteToFile(fd, out.data(), out.length()); CloseFile(fd); VReport(1, " CovDump: %zd caller-callee pairs written\n", total); } // Record the current PC into the event buffer. // Every event is a u32 value (index in tr_pc_array_index) so we compute // it once and then cache in the provided 'cache' storage. // // This function will eventually be inlined by the compiler. void CoverageData::TraceBasicBlock(u32 *id) { // Will trap here if // 1. coverage is not enabled at run-time. // 2. The array tr_event_array is full. *tr_event_pointer = *id - 1; tr_event_pointer++; } void CoverageData::DumpCounters() { if (!common_flags()->coverage_counters) return; uptr n = coverage_data.GetNumberOf8bitCounters(); if (!n) return; InternalScopedBuffer bitset(n); coverage_data.Update8bitCounterBitsetAndClearCounters(bitset.data()); InternalScopedString path(kMaxPathLength); for (uptr m = 0; m < module_name_vec.size(); m++) { auto r = module_name_vec[m]; CHECK(r.copied_module_name); CHECK_LE(r.beg, r.end); CHECK_LE(r.end, size()); const char *base_name = StripModuleName(r.copied_module_name); fd_t fd = CovOpenFile(&path, /* packed */ false, base_name, "counters-sancov"); if (fd == kInvalidFd) return; WriteToFile(fd, bitset.data() + r.beg, r.end - r.beg); CloseFile(fd); VReport(1, " CovDump: %zd counters written for '%s'\n", r.end - r.beg, base_name); } } void CoverageData::DumpAsBitSet() { if (!common_flags()->coverage_bitset) return; if (!size()) return; InternalScopedBuffer out(size()); InternalScopedString path(kMaxPathLength); for (uptr m = 0; m < module_name_vec.size(); m++) { uptr n_set_bits = 0; auto r = module_name_vec[m]; CHECK(r.copied_module_name); CHECK_LE(r.beg, r.end); CHECK_LE(r.end, size()); for (uptr i = r.beg; i < r.end; i++) { uptr pc = UnbundlePc(pc_array[i]); out[i] = pc ? '1' : '0'; if (pc) n_set_bits++; } const char *base_name = StripModuleName(r.copied_module_name); fd_t fd = CovOpenFile(&path, /* packed */false, base_name, "bitset-sancov"); if (fd == kInvalidFd) return; WriteToFile(fd, out.data() + r.beg, r.end - r.beg); CloseFile(fd); VReport(1, " CovDump: bitset of %zd bits written for '%s', %zd bits are set\n", r.end - r.beg, base_name, n_set_bits); } } void CoverageData::GetRangeOffsets(const NamedPcRange& r, Symbolizer* sym, InternalMmapVector* offsets) const { offsets->clear(); for (uptr i = 0; i < kNumWordsForMagic; i++) offsets->push_back(0); CHECK(r.copied_module_name); CHECK_LE(r.beg, r.end); CHECK_LE(r.end, size()); for (uptr i = r.beg; i < r.end; i++) { uptr pc = UnbundlePc(pc_array[i]); uptr counter = UnbundleCounter(pc_array[i]); if (!pc) continue; // Not visited. uptr offset = 0; sym->GetModuleNameAndOffsetForPC(pc, nullptr, &offset); offsets->push_back(BundlePcAndCounter(offset, counter)); } CHECK_GE(offsets->size(), kNumWordsForMagic); SortArray(offsets->data(), offsets->size()); for (uptr i = 0; i < offsets->size(); i++) (*offsets)[i] = UnbundlePc((*offsets)[i]); } static void GenerateHtmlReport(const InternalMmapVector &cov_files) { if (!common_flags()->html_cov_report) { return; } char *sancov_path = FindPathToBinary(common_flags()->sancov_path); if (sancov_path == nullptr) { return; } InternalMmapVector sancov_argv(cov_files.size() * 2 + 3); sancov_argv.push_back(sancov_path); sancov_argv.push_back(internal_strdup("-html-report")); auto argv_deleter = at_scope_exit([&] { for (uptr i = 0; i < sancov_argv.size(); ++i) { InternalFree(sancov_argv[i]); } }); for (const auto &cov_file : cov_files) { sancov_argv.push_back(internal_strdup(cov_file)); } { ListOfModules modules; modules.init(); for (const LoadedModule &module : modules) { sancov_argv.push_back(internal_strdup(module.full_name())); } } InternalScopedString report_path(kMaxPathLength); fd_t report_fd = CovOpenFile(&report_path, false /* packed */, GetProcessName(), "html"); int pid = StartSubprocess(sancov_argv[0], sancov_argv.data(), kInvalidFd /* stdin */, report_fd /* std_out */); if (pid > 0) { int result = WaitForProcess(pid); if (result == 0) Printf("coverage report generated to %s\n", report_path.data()); } } void CoverageData::DumpOffsets() { auto sym = Symbolizer::GetOrInit(); if (!common_flags()->coverage_pcs) return; CHECK_NE(sym, nullptr); InternalMmapVector offsets(0); InternalScopedString path(kMaxPathLength); InternalMmapVector cov_files(module_name_vec.size()); auto cov_files_deleter = at_scope_exit([&] { for (uptr i = 0; i < cov_files.size(); ++i) { InternalFree(cov_files[i]); } }); for (uptr m = 0; m < module_name_vec.size(); m++) { auto r = module_name_vec[m]; GetRangeOffsets(r, sym, &offsets); uptr num_offsets = offsets.size() - kNumWordsForMagic; u64 *magic_p = reinterpret_cast(offsets.data()); CHECK_EQ(*magic_p, 0ULL); // FIXME: we may want to write 32-bit offsets even in 64-mode // if all the offsets are small enough. *magic_p = kMagic; const char *module_name = StripModuleName(r.copied_module_name); if (cov_sandboxed) { if (cov_fd != kInvalidFd) { CovWritePacked(internal_getpid(), module_name, offsets.data(), offsets.size() * sizeof(offsets[0])); VReport(1, " CovDump: %zd PCs written to packed file\n", num_offsets); } } else { // One file per module per process. fd_t fd = CovOpenFile(&path, false /* packed */, module_name); if (fd == kInvalidFd) continue; WriteToFile(fd, offsets.data(), offsets.size() * sizeof(offsets[0])); CloseFile(fd); cov_files.push_back(internal_strdup(path.data())); VReport(1, " CovDump: %s: %zd PCs written\n", path.data(), num_offsets); } } if (cov_fd != kInvalidFd) CloseFile(cov_fd); GenerateHtmlReport(cov_files); } void CoverageData::DumpAll() { if (!coverage_enabled || common_flags()->coverage_direct) return; if (atomic_fetch_add(&dump_once_guard, 1, memory_order_relaxed)) return; DumpAsBitSet(); DumpCounters(); DumpTrace(); DumpOffsets(); DumpCallerCalleePairs(); } void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args) { if (!args) return; if (!coverage_enabled) return; cov_sandboxed = args->coverage_sandboxed; if (!cov_sandboxed) return; cov_max_block_size = args->coverage_max_block_size; if (args->coverage_fd >= 0) { cov_fd = (fd_t)args->coverage_fd; } else { InternalScopedString path(kMaxPathLength); // Pre-open the file now. The sandbox won't allow us to do it later. cov_fd = CovOpenFile(&path, true /* packed */, nullptr); } } fd_t MaybeOpenCovFile(const char *name) { CHECK(name); if (!coverage_enabled) return kInvalidFd; InternalScopedString path(kMaxPathLength); return CovOpenFile(&path, true /* packed */, name); } void CovBeforeFork() { coverage_data.BeforeFork(); } void CovAfterFork(int child_pid) { coverage_data.AfterFork(child_pid); } static void MaybeDumpCoverage() { if (common_flags()->coverage) __sanitizer_cov_dump(); } void InitializeCoverage(bool enabled, const char *dir) { if (coverage_enabled) return; // May happen if two sanitizer enable coverage in the same process. coverage_enabled = enabled; coverage_dir = dir; coverage_data.Init(); if (enabled) coverage_data.Enable(); if (!common_flags()->coverage_direct) Atexit(__sanitizer_cov_dump); AddDieCallback(MaybeDumpCoverage); } void ReInitializeCoverage(bool enabled, const char *dir) { coverage_enabled = enabled; coverage_dir = dir; coverage_data.ReInit(); } void CoverageUpdateMapping() { if (coverage_enabled) CovUpdateMapping(coverage_dir); } } // namespace __sanitizer extern "C" { SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(u32 *guard) { coverage_data.Add(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()), guard); } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_with_check(u32 *guard) { atomic_uint32_t *atomic_guard = reinterpret_cast(guard); if (static_cast( __sanitizer::atomic_load(atomic_guard, memory_order_relaxed)) < 0) __sanitizer_cov(guard); } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_indir_call16(uptr callee, uptr callee_cache16[]) { coverage_data.IndirCall(StackTrace::GetPreviousInstructionPc(GET_CALLER_PC()), callee, callee_cache16, 16); } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init() { coverage_enabled = true; coverage_dir = common_flags()->coverage_dir; coverage_data.Init(); } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { coverage_data.DumpAll(); +#if SANITIZER_LINUX __sanitizer_dump_trace_pc_guard_coverage(); +#endif } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_module_init(s32 *guards, uptr npcs, u8 *counters, const char *comp_unit_name) { coverage_data.InitializeGuards(guards, npcs, comp_unit_name, GET_CALLER_PC()); coverage_data.InitializeCounters(counters, npcs); if (!common_flags()->coverage_direct) return; if (SANITIZER_ANDROID && coverage_enabled) { // dlopen/dlclose interceptors do not work on Android, so we rely on // Extend() calls to update .sancov.map. CovUpdateMapping(coverage_dir, GET_CALLER_PC()); } coverage_data.Extend(npcs); } SANITIZER_INTERFACE_ATTRIBUTE sptr __sanitizer_maybe_open_cov_file(const char *name) { return (sptr)MaybeOpenCovFile(name); } SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_total_unique_coverage() { return atomic_load(&coverage_counter, memory_order_relaxed); } SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_total_unique_caller_callee_pairs() { return atomic_load(&caller_callee_counter, memory_order_relaxed); } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_trace_func_enter(u32 *id) { __sanitizer_cov_with_check(id); coverage_data.TraceBasicBlock(id); } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_trace_basic_block(u32 *id) { __sanitizer_cov_with_check(id); coverage_data.TraceBasicBlock(id); } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_reset_coverage() { ResetGlobalCounters(); coverage_data.ReinitializeGuards(); internal_bzero_aligned16( coverage_data.data(), RoundUpTo(coverage_data.size() * sizeof(coverage_data.data()[0]), 16)); } SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_coverage_guards(uptr **data) { *data = coverage_data.data(); return coverage_data.size(); } SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_number_of_counters() { return coverage_data.GetNumberOf8bitCounters(); } SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_update_counter_bitset_and_clear_counters(u8 *bitset) { return coverage_data.Update8bitCounterBitsetAndClearCounters(bitset); } // Default empty implementations (weak). Users should redefine them. #if !SANITIZER_WINDOWS // weak does not work on Windows. SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_cmp() {} SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_cmp1() {} SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_cmp2() {} SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_cmp4() {} SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_cmp8() {} SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_switch() {} SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_div4() {} SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_div8() {} SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_gep() {} SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_pc_indir() {} #endif // !SANITIZER_WINDOWS } // extern "C" Index: projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.cc (revision 312198) @@ -1,103 +1,128 @@ //===-- sanitizer_libignore.cc --------------------------------------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" #if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC #include "sanitizer_libignore.h" #include "sanitizer_flags.h" #include "sanitizer_posix.h" #include "sanitizer_procmaps.h" namespace __sanitizer { LibIgnore::LibIgnore(LinkerInitialized) { } void LibIgnore::AddIgnoredLibrary(const char *name_templ) { BlockingMutexLock lock(&mutex_); if (count_ >= kMaxLibs) { Report("%s: too many ignored libraries (max: %d)\n", SanitizerToolName, kMaxLibs); Die(); } Lib *lib = &libs_[count_++]; lib->templ = internal_strdup(name_templ); lib->name = nullptr; lib->real_name = nullptr; lib->loaded = false; } void LibIgnore::OnLibraryLoaded(const char *name) { BlockingMutexLock lock(&mutex_); // Try to match suppressions with symlink target. InternalScopedString buf(kMaxPathLength); if (name && internal_readlink(name, buf.data(), buf.size() - 1) > 0 && buf[0]) { for (uptr i = 0; i < count_; i++) { Lib *lib = &libs_[i]; if (!lib->loaded && (!lib->real_name) && TemplateMatch(lib->templ, name)) lib->real_name = internal_strdup(buf.data()); } } // Scan suppressions list and find newly loaded and unloaded libraries. ListOfModules modules; modules.init(); for (uptr i = 0; i < count_; i++) { Lib *lib = &libs_[i]; bool loaded = false; for (const auto &mod : modules) { for (const auto &range : mod.ranges()) { if (!range.executable) continue; if (!TemplateMatch(lib->templ, mod.full_name()) && !(lib->real_name && internal_strcmp(lib->real_name, mod.full_name()) == 0)) continue; if (loaded) { Report("%s: called_from_lib suppression '%s' is matched against" " 2 libraries: '%s' and '%s'\n", SanitizerToolName, lib->templ, lib->name, mod.full_name()); Die(); } loaded = true; if (lib->loaded) continue; VReport(1, "Matched called_from_lib suppression '%s' against library" " '%s'\n", lib->templ, mod.full_name()); lib->loaded = true; lib->name = internal_strdup(mod.full_name()); - const uptr idx = atomic_load(&loaded_count_, memory_order_relaxed); - code_ranges_[idx].begin = range.beg; - code_ranges_[idx].end = range.end; - atomic_store(&loaded_count_, idx + 1, memory_order_release); + const uptr idx = + atomic_load(&ignored_ranges_count_, memory_order_relaxed); + CHECK_LT(idx, kMaxLibs); + ignored_code_ranges_[idx].begin = range.beg; + ignored_code_ranges_[idx].end = range.end; + atomic_store(&ignored_ranges_count_, idx + 1, memory_order_release); break; } } if (lib->loaded && !loaded) { Report("%s: library '%s' that was matched against called_from_lib" " suppression '%s' is unloaded\n", SanitizerToolName, lib->name, lib->templ); Die(); + } + } + + // Track instrumented ranges. + if (track_instrumented_libs_) { + for (const auto &mod : modules) { + if (!mod.instrumented()) + continue; + for (const auto &range : mod.ranges()) { + if (!range.executable) + continue; + if (IsPcInstrumented(range.beg) && IsPcInstrumented(range.end - 1)) + continue; + VReport(1, "Adding instrumented range %p-%p from library '%s'\n", + range.beg, range.end, mod.full_name()); + const uptr idx = + atomic_load(&instrumented_ranges_count_, memory_order_relaxed); + CHECK_LT(idx, kMaxLibs); + instrumented_code_ranges_[idx].begin = range.beg; + instrumented_code_ranges_[idx].end = range.end; + atomic_store(&instrumented_ranges_count_, idx + 1, + memory_order_release); + } } } } void LibIgnore::OnLibraryUnloaded() { OnLibraryLoaded(nullptr); } } // namespace __sanitizer #endif // #if SANITIZER_FREEBSD || SANITIZER_LINUX Index: projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.h =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.h (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_libignore.h (revision 312198) @@ -1,83 +1,114 @@ //===-- sanitizer_libignore.h -----------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // LibIgnore allows to ignore all interceptors called from a particular set // of dynamic libraries. LibIgnore can be initialized with several templates // of names of libraries to be ignored. It finds code ranges for the libraries; // and checks whether the provided PC value belongs to the code ranges. // //===----------------------------------------------------------------------===// #ifndef SANITIZER_LIBIGNORE_H #define SANITIZER_LIBIGNORE_H #include "sanitizer_internal_defs.h" #include "sanitizer_common.h" #include "sanitizer_atomic.h" #include "sanitizer_mutex.h" namespace __sanitizer { class LibIgnore { public: explicit LibIgnore(LinkerInitialized); // Must be called during initialization. void AddIgnoredLibrary(const char *name_templ); + void IgnoreNoninstrumentedModules(bool enable) { + track_instrumented_libs_ = enable; + } // Must be called after a new dynamic library is loaded. void OnLibraryLoaded(const char *name); // Must be called after a dynamic library is unloaded. void OnLibraryUnloaded(); - // Checks whether the provided PC belongs to one of the ignored libraries. - bool IsIgnored(uptr pc) const; + // Checks whether the provided PC belongs to one of the ignored libraries or + // the PC should be ignored because it belongs to an non-instrumented module + // (when ignore_noninstrumented_modules=1). Also returns true via + // "pc_in_ignored_lib" if the PC is in an ignored library, false otherwise. + bool IsIgnored(uptr pc, bool *pc_in_ignored_lib) const; + // Checks whether the provided PC belongs to an instrumented module. + bool IsPcInstrumented(uptr pc) const; + private: struct Lib { char *templ; char *name; char *real_name; // target of symlink bool loaded; }; struct LibCodeRange { uptr begin; uptr end; }; + inline bool IsInRange(uptr pc, const LibCodeRange &range) const { + return (pc >= range.begin && pc < range.end); + } + static const uptr kMaxLibs = 128; // Hot part: - atomic_uintptr_t loaded_count_; - LibCodeRange code_ranges_[kMaxLibs]; + atomic_uintptr_t ignored_ranges_count_; + LibCodeRange ignored_code_ranges_[kMaxLibs]; + atomic_uintptr_t instrumented_ranges_count_; + LibCodeRange instrumented_code_ranges_[kMaxLibs]; + // Cold part: BlockingMutex mutex_; uptr count_; Lib libs_[kMaxLibs]; + bool track_instrumented_libs_; // Disallow copying of LibIgnore objects. LibIgnore(const LibIgnore&); // not implemented void operator = (const LibIgnore&); // not implemented }; -inline bool LibIgnore::IsIgnored(uptr pc) const { - const uptr n = atomic_load(&loaded_count_, memory_order_acquire); +inline bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const { + const uptr n = atomic_load(&ignored_ranges_count_, memory_order_acquire); for (uptr i = 0; i < n; i++) { - if (pc >= code_ranges_[i].begin && pc < code_ranges_[i].end) + if (IsInRange(pc, ignored_code_ranges_[i])) { + *pc_in_ignored_lib = true; + return true; + } + } + *pc_in_ignored_lib = false; + if (track_instrumented_libs_ && !IsPcInstrumented(pc)) + return true; + return false; +} + +inline bool LibIgnore::IsPcInstrumented(uptr pc) const { + const uptr n = atomic_load(&instrumented_ranges_count_, memory_order_acquire); + for (uptr i = 0; i < n; i++) { + if (IsInRange(pc, instrumented_code_ranges_[i])) return true; } return false; } } // namespace __sanitizer #endif // SANITIZER_LIBIGNORE_H Index: projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h (revision 312198) @@ -1,1484 +1,1487 @@ //===-- sanitizer_platform_limits_posix.h ---------------------------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file is a part of Sanitizer common code. // // Sizes and layouts of platform-specific POSIX data structures. //===----------------------------------------------------------------------===// #ifndef SANITIZER_PLATFORM_LIMITS_POSIX_H #define SANITIZER_PLATFORM_LIMITS_POSIX_H #include "sanitizer_internal_defs.h" #include "sanitizer_platform.h" #if SANITIZER_FREEBSD // FreeBSD's dlopen() returns a pointer to an Obj_Entry structure that // incorporates the map structure. # define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) \ ((link_map*)((handle) == nullptr ? nullptr : ((char*)(handle) + 544))) #else # define GET_LINK_MAP_BY_DLOPEN_HANDLE(handle) ((link_map*)(handle)) #endif // !SANITIZER_FREEBSD #ifndef __GLIBC_PREREQ #define __GLIBC_PREREQ(x, y) 0 #endif namespace __sanitizer { extern unsigned struct_utsname_sz; extern unsigned struct_stat_sz; #if !SANITIZER_FREEBSD && !SANITIZER_IOS extern unsigned struct_stat64_sz; #endif extern unsigned struct_rusage_sz; extern unsigned siginfo_t_sz; extern unsigned struct_itimerval_sz; extern unsigned pthread_t_sz; extern unsigned pthread_cond_t_sz; extern unsigned pid_t_sz; extern unsigned timeval_sz; extern unsigned uid_t_sz; extern unsigned gid_t_sz; extern unsigned mbstate_t_sz; extern unsigned struct_timezone_sz; extern unsigned struct_tms_sz; extern unsigned struct_itimerspec_sz; extern unsigned struct_sigevent_sz; extern unsigned struct_sched_param_sz; extern unsigned struct_statfs64_sz; #if !SANITIZER_ANDROID extern unsigned struct_statfs_sz; extern unsigned struct_sockaddr_sz; extern unsigned ucontext_t_sz; #endif // !SANITIZER_ANDROID #if SANITIZER_LINUX #if defined(__x86_64__) const unsigned struct_kernel_stat_sz = 144; const unsigned struct_kernel_stat64_sz = 0; #elif defined(__i386__) const unsigned struct_kernel_stat_sz = 64; const unsigned struct_kernel_stat64_sz = 96; #elif defined(__arm__) const unsigned struct_kernel_stat_sz = 64; const unsigned struct_kernel_stat64_sz = 104; #elif defined(__aarch64__) const unsigned struct_kernel_stat_sz = 128; const unsigned struct_kernel_stat64_sz = 104; #elif defined(__powerpc__) && !defined(__powerpc64__) const unsigned struct_kernel_stat_sz = 72; const unsigned struct_kernel_stat64_sz = 104; #elif defined(__powerpc64__) const unsigned struct_kernel_stat_sz = 144; const unsigned struct_kernel_stat64_sz = 104; #elif defined(__riscv__) /* RISCVTODO: check that these values are correct */ const unsigned struct_kernel_stat_sz = 128; const unsigned struct_kernel_stat64_sz = 128; #elif defined(__mips__) const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID ? FIRST_32_SECOND_64(104, 128) : FIRST_32_SECOND_64(144, 216); const unsigned struct_kernel_stat64_sz = 104; #elif defined(__s390__) && !defined(__s390x__) const unsigned struct_kernel_stat_sz = 64; const unsigned struct_kernel_stat64_sz = 104; #elif defined(__s390x__) const unsigned struct_kernel_stat_sz = 144; const unsigned struct_kernel_stat64_sz = 0; #elif defined(__sparc__) && defined(__arch64__) const unsigned struct___old_kernel_stat_sz = 0; const unsigned struct_kernel_stat_sz = 104; const unsigned struct_kernel_stat64_sz = 144; #elif defined(__sparc__) && !defined(__arch64__) const unsigned struct___old_kernel_stat_sz = 0; const unsigned struct_kernel_stat_sz = 64; const unsigned struct_kernel_stat64_sz = 104; #endif struct __sanitizer_perf_event_attr { unsigned type; unsigned size; // More fields that vary with the kernel version. }; extern unsigned struct_epoll_event_sz; extern unsigned struct_sysinfo_sz; extern unsigned __user_cap_header_struct_sz; extern unsigned __user_cap_data_struct_sz; extern unsigned struct_new_utsname_sz; extern unsigned struct_old_utsname_sz; extern unsigned struct_oldold_utsname_sz; const unsigned struct_kexec_segment_sz = 4 * sizeof(unsigned long); #endif // SANITIZER_LINUX #if SANITIZER_LINUX || SANITIZER_FREEBSD #if defined(__powerpc64__) || defined(__riscv__) || defined(__s390__) const unsigned struct___old_kernel_stat_sz = 0; #elif !defined(__sparc__) const unsigned struct___old_kernel_stat_sz = 32; #endif extern unsigned struct_rlimit_sz; extern unsigned struct_utimbuf_sz; extern unsigned struct_timespec_sz; struct __sanitizer_iocb { u64 aio_data; u32 aio_key_or_aio_reserved1; // Simply crazy. u32 aio_reserved1_or_aio_key; // Luckily, we don't need these. u16 aio_lio_opcode; s16 aio_reqprio; u32 aio_fildes; u64 aio_buf; u64 aio_nbytes; s64 aio_offset; u64 aio_reserved2; u64 aio_reserved3; }; struct __sanitizer_io_event { u64 data; u64 obj; u64 res; u64 res2; }; const unsigned iocb_cmd_pread = 0; const unsigned iocb_cmd_pwrite = 1; const unsigned iocb_cmd_preadv = 7; const unsigned iocb_cmd_pwritev = 8; struct __sanitizer___sysctl_args { int *name; int nlen; void *oldval; uptr *oldlenp; void *newval; uptr newlen; unsigned long ___unused[4]; }; const unsigned old_sigset_t_sz = sizeof(unsigned long); struct __sanitizer_sem_t { #if SANITIZER_ANDROID && defined(_LP64) int data[4]; #elif SANITIZER_ANDROID && !defined(_LP64) int data; #elif SANITIZER_LINUX uptr data[4]; #elif SANITIZER_FREEBSD u32 data[4]; #endif }; #endif // SANITIZER_LINUX || SANITIZER_FREEBSD #if SANITIZER_ANDROID struct __sanitizer_mallinfo { uptr v[10]; }; #endif #if SANITIZER_LINUX && !SANITIZER_ANDROID struct __sanitizer_mallinfo { int v[10]; }; extern unsigned struct_ustat_sz; extern unsigned struct_rlimit64_sz; extern unsigned struct_statvfs64_sz; struct __sanitizer_ipc_perm { int __key; int uid; int gid; int cuid; int cgid; #ifdef __powerpc__ unsigned mode; unsigned __seq; u64 __unused1; u64 __unused2; #elif defined(__sparc__) #if defined(__arch64__) unsigned mode; unsigned short __pad1; #else unsigned short __pad1; unsigned short mode; unsigned short __pad2; #endif unsigned short __seq; unsigned long long __unused1; unsigned long long __unused2; #elif defined(__mips__) || defined(__aarch64__) || defined(__s390x__) unsigned int mode; unsigned short __seq; unsigned short __pad1; unsigned long __unused1; unsigned long __unused2; #else unsigned short mode; unsigned short __pad1; unsigned short __seq; unsigned short __pad2; #if defined(__x86_64__) && !defined(_LP64) u64 __unused1; u64 __unused2; #else unsigned long __unused1; unsigned long __unused2; #endif #endif }; struct __sanitizer_shmid_ds { __sanitizer_ipc_perm shm_perm; #if defined(__sparc__) #if !defined(__arch64__) u32 __pad1; #endif long shm_atime; #if !defined(__arch64__) u32 __pad2; #endif long shm_dtime; #if !defined(__arch64__) u32 __pad3; #endif long shm_ctime; uptr shm_segsz; int shm_cpid; int shm_lpid; unsigned long shm_nattch; unsigned long __glibc_reserved1; unsigned long __glibc_reserved2; #else #ifndef __powerpc__ uptr shm_segsz; #elif !defined(__powerpc64__) uptr __unused0; #endif #if defined(__x86_64__) && !defined(_LP64) u64 shm_atime; u64 shm_dtime; u64 shm_ctime; #else uptr shm_atime; #if !defined(_LP64) && !defined(__mips__) uptr __unused1; #endif uptr shm_dtime; #if !defined(_LP64) && !defined(__mips__) uptr __unused2; #endif uptr shm_ctime; #if !defined(_LP64) && !defined(__mips__) uptr __unused3; #endif #endif #ifdef __powerpc__ uptr shm_segsz; #endif int shm_cpid; int shm_lpid; #if defined(__x86_64__) && !defined(_LP64) u64 shm_nattch; u64 __unused4; u64 __unused5; #else uptr shm_nattch; uptr __unused4; uptr __unused5; #endif #endif }; #elif SANITIZER_FREEBSD struct __sanitizer_ipc_perm { unsigned int cuid; unsigned int cgid; unsigned int uid; unsigned int gid; unsigned short mode; unsigned short seq; long key; }; struct __sanitizer_shmid_ds { __sanitizer_ipc_perm shm_perm; unsigned long shm_segsz; unsigned int shm_lpid; unsigned int shm_cpid; int shm_nattch; unsigned long shm_atime; unsigned long shm_dtime; unsigned long shm_ctime; }; #endif #if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID extern unsigned struct_msqid_ds_sz; extern unsigned struct_mq_attr_sz; extern unsigned struct_timex_sz; extern unsigned struct_statvfs_sz; #endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID struct __sanitizer_iovec { void *iov_base; uptr iov_len; }; #if !SANITIZER_ANDROID struct __sanitizer_ifaddrs { struct __sanitizer_ifaddrs *ifa_next; char *ifa_name; unsigned int ifa_flags; void *ifa_addr; // (struct sockaddr *) void *ifa_netmask; // (struct sockaddr *) // This is a union on Linux. # ifdef ifa_dstaddr # undef ifa_dstaddr # endif void *ifa_dstaddr; // (struct sockaddr *) void *ifa_data; }; #endif // !SANITIZER_ANDROID #if SANITIZER_MAC typedef unsigned long __sanitizer_pthread_key_t; #else typedef unsigned __sanitizer_pthread_key_t; #endif #if SANITIZER_LINUX && !SANITIZER_ANDROID struct __sanitizer_XDR { int x_op; void *x_ops; uptr x_public; uptr x_private; uptr x_base; unsigned x_handy; }; const int __sanitizer_XDR_ENCODE = 0; const int __sanitizer_XDR_DECODE = 1; const int __sanitizer_XDR_FREE = 2; #endif struct __sanitizer_passwd { char *pw_name; char *pw_passwd; int pw_uid; int pw_gid; #if SANITIZER_MAC || SANITIZER_FREEBSD long pw_change; char *pw_class; #endif #if !(SANITIZER_ANDROID && (SANITIZER_WORDSIZE == 32)) char *pw_gecos; #endif char *pw_dir; char *pw_shell; #if SANITIZER_MAC || SANITIZER_FREEBSD long pw_expire; #endif #if SANITIZER_FREEBSD int pw_fields; #endif }; struct __sanitizer_group { char *gr_name; char *gr_passwd; int gr_gid; char **gr_mem; }; #if defined(__x86_64__) && !defined(_LP64) typedef long long __sanitizer_time_t; #else typedef long __sanitizer_time_t; #endif struct __sanitizer_timeb { __sanitizer_time_t time; unsigned short millitm; short timezone; short dstflag; }; struct __sanitizer_ether_addr { u8 octet[6]; }; struct __sanitizer_tm { int tm_sec; int tm_min; int tm_hour; int tm_mday; int tm_mon; int tm_year; int tm_wday; int tm_yday; int tm_isdst; long int tm_gmtoff; const char *tm_zone; }; #if SANITIZER_LINUX struct __sanitizer_mntent { char *mnt_fsname; char *mnt_dir; char *mnt_type; char *mnt_opts; int mnt_freq; int mnt_passno; }; #endif #if SANITIZER_MAC || SANITIZER_FREEBSD struct __sanitizer_msghdr { void *msg_name; unsigned msg_namelen; struct __sanitizer_iovec *msg_iov; unsigned msg_iovlen; void *msg_control; unsigned msg_controllen; int msg_flags; }; struct __sanitizer_cmsghdr { unsigned cmsg_len; int cmsg_level; int cmsg_type; }; #else struct __sanitizer_msghdr { void *msg_name; unsigned msg_namelen; struct __sanitizer_iovec *msg_iov; uptr msg_iovlen; void *msg_control; uptr msg_controllen; int msg_flags; }; struct __sanitizer_cmsghdr { uptr cmsg_len; int cmsg_level; int cmsg_type; }; #endif #if SANITIZER_MAC struct __sanitizer_dirent { unsigned long long d_ino; unsigned long long d_seekoff; unsigned short d_reclen; // more fields that we don't care about }; #elif SANITIZER_FREEBSD struct __sanitizer_dirent { unsigned int d_fileno; unsigned short d_reclen; // more fields that we don't care about }; #elif SANITIZER_ANDROID || defined(__x86_64__) struct __sanitizer_dirent { unsigned long long d_ino; unsigned long long d_off; unsigned short d_reclen; // more fields that we don't care about }; #else struct __sanitizer_dirent { uptr d_ino; uptr d_off; unsigned short d_reclen; // more fields that we don't care about }; #endif #if SANITIZER_LINUX && !SANITIZER_ANDROID struct __sanitizer_dirent64 { unsigned long long d_ino; unsigned long long d_off; unsigned short d_reclen; // more fields that we don't care about }; #endif // 'clock_t' is 32 bits wide on x64 FreeBSD #if SANITIZER_FREEBSD typedef int __sanitizer_clock_t; #elif defined(__x86_64__) && !defined(_LP64) typedef long long __sanitizer_clock_t; #else typedef long __sanitizer_clock_t; #endif #if SANITIZER_LINUX typedef int __sanitizer_clockid_t; #endif #if SANITIZER_LINUX || SANITIZER_FREEBSD #if defined(_LP64) || defined(__x86_64__) || defined(__powerpc__)\ || defined(__mips__) typedef unsigned __sanitizer___kernel_uid_t; typedef unsigned __sanitizer___kernel_gid_t; #else typedef unsigned short __sanitizer___kernel_uid_t; typedef unsigned short __sanitizer___kernel_gid_t; #endif #if defined(__x86_64__) && !defined(_LP64) typedef long long __sanitizer___kernel_off_t; #else typedef long __sanitizer___kernel_off_t; #endif #if defined(__powerpc__) || defined(__mips__) || defined(__riscv__) typedef unsigned int __sanitizer___kernel_old_uid_t; typedef unsigned int __sanitizer___kernel_old_gid_t; #else typedef unsigned short __sanitizer___kernel_old_uid_t; typedef unsigned short __sanitizer___kernel_old_gid_t; #endif typedef long long __sanitizer___kernel_loff_t; typedef struct { unsigned long fds_bits[1024 / (8 * sizeof(long))]; } __sanitizer___kernel_fd_set; #endif // This thing depends on the platform. We are only interested in the upper // limit. Verified with a compiler assert in .cc. const int pthread_attr_t_max_sz = 128; union __sanitizer_pthread_attr_t { char size[pthread_attr_t_max_sz]; // NOLINT void *align; }; #if SANITIZER_ANDROID # if SANITIZER_MIPS typedef unsigned long __sanitizer_sigset_t[16/sizeof(unsigned long)]; # else typedef unsigned long __sanitizer_sigset_t; # endif #elif SANITIZER_MAC typedef unsigned __sanitizer_sigset_t; #elif SANITIZER_LINUX struct __sanitizer_sigset_t { // The size is determined by looking at sizeof of real sigset_t on linux. uptr val[128 / sizeof(uptr)]; }; #elif SANITIZER_FREEBSD struct __sanitizer_sigset_t { // uint32_t * 4 unsigned int __bits[4]; }; #endif // Linux system headers define the 'sa_handler' and 'sa_sigaction' macros. #if SANITIZER_ANDROID && (SANITIZER_WORDSIZE == 64) struct __sanitizer_sigaction { unsigned sa_flags; union { void (*sigaction)(int sig, void *siginfo, void *uctx); void (*handler)(int sig); }; __sanitizer_sigset_t sa_mask; void (*sa_restorer)(); }; #elif SANITIZER_ANDROID && SANITIZER_MIPS32 // check this before WORDSIZE == 32 struct __sanitizer_sigaction { unsigned sa_flags; union { void (*sigaction)(int sig, void *siginfo, void *uctx); void (*handler)(int sig); }; __sanitizer_sigset_t sa_mask; }; #elif SANITIZER_ANDROID && (SANITIZER_WORDSIZE == 32) struct __sanitizer_sigaction { union { void (*sigaction)(int sig, void *siginfo, void *uctx); void (*handler)(int sig); }; __sanitizer_sigset_t sa_mask; uptr sa_flags; void (*sa_restorer)(); }; #else // !SANITIZER_ANDROID struct __sanitizer_sigaction { #if defined(__mips__) && !SANITIZER_FREEBSD unsigned int sa_flags; #endif union { void (*sigaction)(int sig, void *siginfo, void *uctx); void (*handler)(int sig); }; #if SANITIZER_FREEBSD int sa_flags; __sanitizer_sigset_t sa_mask; #else #if defined(__s390x__) int sa_resv; #else __sanitizer_sigset_t sa_mask; #endif #ifndef __mips__ #if defined(__sparc__) #if __GLIBC_PREREQ (2, 20) - // On sparc glibc 2.19 and earlier sa_flags was unsigned long, and - // __glibc_reserved0 didn't exist. + // On sparc glibc 2.19 and earlier sa_flags was unsigned long. +#if defined(__arch64__) + // To maintain ABI compatibility on sparc64 when switching to an int, + // __glibc_reserved0 was added. int __glibc_reserved0; +#endif int sa_flags; #else unsigned long sa_flags; #endif #else int sa_flags; #endif #endif #endif #if SANITIZER_LINUX void (*sa_restorer)(); #endif #if defined(__mips__) && (SANITIZER_WORDSIZE == 32) int sa_resv[1]; #endif #if defined(__s390x__) __sanitizer_sigset_t sa_mask; #endif }; #endif // !SANITIZER_ANDROID #if SANITIZER_FREEBSD typedef __sanitizer_sigset_t __sanitizer_kernel_sigset_t; #elif defined(__mips__) struct __sanitizer_kernel_sigset_t { uptr sig[2]; }; #else struct __sanitizer_kernel_sigset_t { u8 sig[8]; }; #endif // Linux system headers define the 'sa_handler' and 'sa_sigaction' macros. #if SANITIZER_MIPS struct __sanitizer_kernel_sigaction_t { unsigned int sa_flags; union { void (*handler)(int signo); void (*sigaction)(int signo, void *info, void *ctx); }; __sanitizer_kernel_sigset_t sa_mask; void (*sa_restorer)(void); }; #else struct __sanitizer_kernel_sigaction_t { union { void (*handler)(int signo); void (*sigaction)(int signo, void *info, void *ctx); }; unsigned long sa_flags; void (*sa_restorer)(void); __sanitizer_kernel_sigset_t sa_mask; }; #endif extern uptr sig_ign; extern uptr sig_dfl; extern uptr sa_siginfo; #if SANITIZER_LINUX extern int e_tabsz; #endif extern int af_inet; extern int af_inet6; uptr __sanitizer_in_addr_sz(int af); #if SANITIZER_LINUX || SANITIZER_FREEBSD struct __sanitizer_dl_phdr_info { uptr dlpi_addr; const char *dlpi_name; const void *dlpi_phdr; short dlpi_phnum; }; extern unsigned struct_ElfW_Phdr_sz; #endif struct __sanitizer_addrinfo { int ai_flags; int ai_family; int ai_socktype; int ai_protocol; #if SANITIZER_ANDROID || SANITIZER_MAC || SANITIZER_FREEBSD unsigned ai_addrlen; char *ai_canonname; void *ai_addr; #else // LINUX unsigned ai_addrlen; void *ai_addr; char *ai_canonname; #endif struct __sanitizer_addrinfo *ai_next; }; struct __sanitizer_hostent { char *h_name; char **h_aliases; int h_addrtype; int h_length; char **h_addr_list; }; struct __sanitizer_pollfd { int fd; short events; short revents; }; #if SANITIZER_ANDROID || SANITIZER_MAC || SANITIZER_FREEBSD typedef unsigned __sanitizer_nfds_t; #else typedef unsigned long __sanitizer_nfds_t; #endif #if !SANITIZER_ANDROID # if SANITIZER_LINUX struct __sanitizer_glob_t { uptr gl_pathc; char **gl_pathv; uptr gl_offs; int gl_flags; void (*gl_closedir)(void *dirp); void *(*gl_readdir)(void *dirp); void *(*gl_opendir)(const char *); int (*gl_lstat)(const char *, void *); int (*gl_stat)(const char *, void *); }; # elif SANITIZER_FREEBSD struct __sanitizer_glob_t { uptr gl_pathc; uptr gl_matchc; uptr gl_offs; int gl_flags; char **gl_pathv; int (*gl_errfunc)(const char*, int); void (*gl_closedir)(void *dirp); struct dirent *(*gl_readdir)(void *dirp); void *(*gl_opendir)(const char*); int (*gl_lstat)(const char*, void* /* struct stat* */); int (*gl_stat)(const char*, void* /* struct stat* */); }; # endif // SANITIZER_FREEBSD # if SANITIZER_LINUX || SANITIZER_FREEBSD extern int glob_nomatch; extern int glob_altdirfunc; # endif #endif // !SANITIZER_ANDROID extern unsigned path_max; struct __sanitizer_wordexp_t { uptr we_wordc; char **we_wordv; uptr we_offs; #if SANITIZER_FREEBSD char *we_strings; uptr we_nbytes; #endif }; #if SANITIZER_LINUX && !SANITIZER_ANDROID struct __sanitizer_FILE { int _flags; char *_IO_read_ptr; char *_IO_read_end; char *_IO_read_base; char *_IO_write_base; char *_IO_write_ptr; char *_IO_write_end; char *_IO_buf_base; char *_IO_buf_end; char *_IO_save_base; char *_IO_backup_base; char *_IO_save_end; void *_markers; __sanitizer_FILE *_chain; int _fileno; }; # define SANITIZER_HAS_STRUCT_FILE 1 #else typedef void __sanitizer_FILE; # define SANITIZER_HAS_STRUCT_FILE 0 #endif #if SANITIZER_LINUX && !SANITIZER_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__) || \ defined(__s390__)) extern unsigned struct_user_regs_struct_sz; extern unsigned struct_user_fpregs_struct_sz; extern unsigned struct_user_fpxregs_struct_sz; extern unsigned struct_user_vfpregs_struct_sz; extern int ptrace_peektext; extern int ptrace_peekdata; extern int ptrace_peekuser; extern int ptrace_getregs; extern int ptrace_setregs; extern int ptrace_getfpregs; extern int ptrace_setfpregs; extern int ptrace_getfpxregs; extern int ptrace_setfpxregs; extern int ptrace_getvfpregs; extern int ptrace_setvfpregs; extern int ptrace_getsiginfo; extern int ptrace_setsiginfo; extern int ptrace_getregset; extern int ptrace_setregset; extern int ptrace_geteventmsg; #endif #if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID extern unsigned struct_shminfo_sz; extern unsigned struct_shm_info_sz; extern int shmctl_ipc_stat; extern int shmctl_ipc_info; extern int shmctl_shm_info; extern int shmctl_shm_stat; #endif #if !SANITIZER_MAC && !SANITIZER_FREEBSD extern unsigned struct_utmp_sz; #endif #if !SANITIZER_ANDROID extern unsigned struct_utmpx_sz; #endif extern int map_fixed; // ioctl arguments struct __sanitizer_ifconf { int ifc_len; union { void *ifcu_req; } ifc_ifcu; #if SANITIZER_MAC } __attribute__((packed)); #else }; #endif #if SANITIZER_LINUX && !SANITIZER_ANDROID struct __sanitizer__obstack_chunk { char *limit; struct __sanitizer__obstack_chunk *prev; }; struct __sanitizer_obstack { long chunk_size; struct __sanitizer__obstack_chunk *chunk; char *object_base; char *next_free; uptr more_fields[7]; }; typedef uptr (*__sanitizer_cookie_io_read)(void *cookie, char *buf, uptr size); typedef uptr (*__sanitizer_cookie_io_write)(void *cookie, const char *buf, uptr size); typedef int (*__sanitizer_cookie_io_seek)(void *cookie, u64 *offset, int whence); typedef int (*__sanitizer_cookie_io_close)(void *cookie); struct __sanitizer_cookie_io_functions_t { __sanitizer_cookie_io_read read; __sanitizer_cookie_io_write write; __sanitizer_cookie_io_seek seek; __sanitizer_cookie_io_close close; }; #endif #define IOC_NRBITS 8 #define IOC_TYPEBITS 8 #if defined(__powerpc__) || defined(__powerpc64__) || defined(__mips__) || \ defined(__sparc__) #define IOC_SIZEBITS 13 #define IOC_DIRBITS 3 #define IOC_NONE 1U #define IOC_WRITE 4U #define IOC_READ 2U #else #define IOC_SIZEBITS 14 #define IOC_DIRBITS 2 #define IOC_NONE 0U #define IOC_WRITE 1U #define IOC_READ 2U #endif #define IOC_NRMASK ((1 << IOC_NRBITS) - 1) #define IOC_TYPEMASK ((1 << IOC_TYPEBITS) - 1) #define IOC_SIZEMASK ((1 << IOC_SIZEBITS) - 1) #if defined(IOC_DIRMASK) #undef IOC_DIRMASK #endif #define IOC_DIRMASK ((1 << IOC_DIRBITS) - 1) #define IOC_NRSHIFT 0 #define IOC_TYPESHIFT (IOC_NRSHIFT + IOC_NRBITS) #define IOC_SIZESHIFT (IOC_TYPESHIFT + IOC_TYPEBITS) #define IOC_DIRSHIFT (IOC_SIZESHIFT + IOC_SIZEBITS) #define EVIOC_EV_MAX 0x1f #define EVIOC_ABS_MAX 0x3f #define IOC_DIR(nr) (((nr) >> IOC_DIRSHIFT) & IOC_DIRMASK) #define IOC_TYPE(nr) (((nr) >> IOC_TYPESHIFT) & IOC_TYPEMASK) #define IOC_NR(nr) (((nr) >> IOC_NRSHIFT) & IOC_NRMASK) #if defined(__sparc__) // In sparc the 14 bits SIZE field overlaps with the // least significant bit of DIR, so either IOC_READ or // IOC_WRITE shall be 1 in order to get a non-zero SIZE. #define IOC_SIZE(nr) \ ((((((nr) >> 29) & 0x7) & (4U | 2U)) == 0) ? 0 : (((nr) >> 16) & 0x3fff)) #else #define IOC_SIZE(nr) (((nr) >> IOC_SIZESHIFT) & IOC_SIZEMASK) #endif extern unsigned struct_ifreq_sz; extern unsigned struct_termios_sz; extern unsigned struct_winsize_sz; #if SANITIZER_LINUX extern unsigned struct_arpreq_sz; extern unsigned struct_cdrom_msf_sz; extern unsigned struct_cdrom_multisession_sz; extern unsigned struct_cdrom_read_audio_sz; extern unsigned struct_cdrom_subchnl_sz; extern unsigned struct_cdrom_ti_sz; extern unsigned struct_cdrom_tocentry_sz; extern unsigned struct_cdrom_tochdr_sz; extern unsigned struct_cdrom_volctrl_sz; extern unsigned struct_ff_effect_sz; extern unsigned struct_floppy_drive_params_sz; extern unsigned struct_floppy_drive_struct_sz; extern unsigned struct_floppy_fdc_state_sz; extern unsigned struct_floppy_max_errors_sz; extern unsigned struct_floppy_raw_cmd_sz; extern unsigned struct_floppy_struct_sz; extern unsigned struct_floppy_write_errors_sz; extern unsigned struct_format_descr_sz; extern unsigned struct_hd_driveid_sz; extern unsigned struct_hd_geometry_sz; extern unsigned struct_input_absinfo_sz; extern unsigned struct_input_id_sz; extern unsigned struct_mtpos_sz; extern unsigned struct_termio_sz; extern unsigned struct_vt_consize_sz; extern unsigned struct_vt_sizes_sz; extern unsigned struct_vt_stat_sz; #endif // SANITIZER_LINUX #if SANITIZER_LINUX || SANITIZER_FREEBSD extern unsigned struct_copr_buffer_sz; extern unsigned struct_copr_debug_buf_sz; extern unsigned struct_copr_msg_sz; extern unsigned struct_midi_info_sz; extern unsigned struct_mtget_sz; extern unsigned struct_mtop_sz; extern unsigned struct_rtentry_sz; extern unsigned struct_sbi_instrument_sz; extern unsigned struct_seq_event_rec_sz; extern unsigned struct_synth_info_sz; extern unsigned struct_vt_mode_sz; #endif // SANITIZER_LINUX || SANITIZER_FREEBSD #if SANITIZER_LINUX && !SANITIZER_ANDROID extern unsigned struct_ax25_parms_struct_sz; extern unsigned struct_cyclades_monitor_sz; extern unsigned struct_input_keymap_entry_sz; extern unsigned struct_ipx_config_data_sz; extern unsigned struct_kbdiacrs_sz; extern unsigned struct_kbentry_sz; extern unsigned struct_kbkeycode_sz; extern unsigned struct_kbsentry_sz; extern unsigned struct_mtconfiginfo_sz; extern unsigned struct_nr_parms_struct_sz; extern unsigned struct_scc_modem_sz; extern unsigned struct_scc_stat_sz; extern unsigned struct_serial_multiport_struct_sz; extern unsigned struct_serial_struct_sz; extern unsigned struct_sockaddr_ax25_sz; extern unsigned struct_unimapdesc_sz; extern unsigned struct_unimapinit_sz; #endif // SANITIZER_LINUX && !SANITIZER_ANDROID #if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID extern unsigned struct_audio_buf_info_sz; extern unsigned struct_ppp_stats_sz; #endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID #if !SANITIZER_ANDROID && !SANITIZER_MAC extern unsigned struct_sioc_sg_req_sz; extern unsigned struct_sioc_vif_req_sz; #endif // ioctl request identifiers // A special value to mark ioctls that are not present on the target platform, // when it can not be determined without including any system headers. extern const unsigned IOCTL_NOT_PRESENT; extern unsigned IOCTL_FIOASYNC; extern unsigned IOCTL_FIOCLEX; extern unsigned IOCTL_FIOGETOWN; extern unsigned IOCTL_FIONBIO; extern unsigned IOCTL_FIONCLEX; extern unsigned IOCTL_FIOSETOWN; extern unsigned IOCTL_SIOCADDMULTI; extern unsigned IOCTL_SIOCATMARK; extern unsigned IOCTL_SIOCDELMULTI; extern unsigned IOCTL_SIOCGIFADDR; extern unsigned IOCTL_SIOCGIFBRDADDR; extern unsigned IOCTL_SIOCGIFCONF; extern unsigned IOCTL_SIOCGIFDSTADDR; extern unsigned IOCTL_SIOCGIFFLAGS; extern unsigned IOCTL_SIOCGIFMETRIC; extern unsigned IOCTL_SIOCGIFMTU; extern unsigned IOCTL_SIOCGIFNETMASK; extern unsigned IOCTL_SIOCGPGRP; extern unsigned IOCTL_SIOCSIFADDR; extern unsigned IOCTL_SIOCSIFBRDADDR; extern unsigned IOCTL_SIOCSIFDSTADDR; extern unsigned IOCTL_SIOCSIFFLAGS; extern unsigned IOCTL_SIOCSIFMETRIC; extern unsigned IOCTL_SIOCSIFMTU; extern unsigned IOCTL_SIOCSIFNETMASK; extern unsigned IOCTL_SIOCSPGRP; extern unsigned IOCTL_TIOCCONS; extern unsigned IOCTL_TIOCEXCL; extern unsigned IOCTL_TIOCGETD; extern unsigned IOCTL_TIOCGPGRP; extern unsigned IOCTL_TIOCGWINSZ; extern unsigned IOCTL_TIOCMBIC; extern unsigned IOCTL_TIOCMBIS; extern unsigned IOCTL_TIOCMGET; extern unsigned IOCTL_TIOCMSET; extern unsigned IOCTL_TIOCNOTTY; extern unsigned IOCTL_TIOCNXCL; extern unsigned IOCTL_TIOCOUTQ; extern unsigned IOCTL_TIOCPKT; extern unsigned IOCTL_TIOCSCTTY; extern unsigned IOCTL_TIOCSETD; extern unsigned IOCTL_TIOCSPGRP; extern unsigned IOCTL_TIOCSTI; extern unsigned IOCTL_TIOCSWINSZ; #if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID extern unsigned IOCTL_SIOCGETSGCNT; extern unsigned IOCTL_SIOCGETVIFCNT; #endif #if SANITIZER_LINUX extern unsigned IOCTL_EVIOCGABS; extern unsigned IOCTL_EVIOCGBIT; extern unsigned IOCTL_EVIOCGEFFECTS; extern unsigned IOCTL_EVIOCGID; extern unsigned IOCTL_EVIOCGKEY; extern unsigned IOCTL_EVIOCGKEYCODE; extern unsigned IOCTL_EVIOCGLED; extern unsigned IOCTL_EVIOCGNAME; extern unsigned IOCTL_EVIOCGPHYS; extern unsigned IOCTL_EVIOCGRAB; extern unsigned IOCTL_EVIOCGREP; extern unsigned IOCTL_EVIOCGSND; extern unsigned IOCTL_EVIOCGSW; extern unsigned IOCTL_EVIOCGUNIQ; extern unsigned IOCTL_EVIOCGVERSION; extern unsigned IOCTL_EVIOCRMFF; extern unsigned IOCTL_EVIOCSABS; extern unsigned IOCTL_EVIOCSFF; extern unsigned IOCTL_EVIOCSKEYCODE; extern unsigned IOCTL_EVIOCSREP; extern unsigned IOCTL_BLKFLSBUF; extern unsigned IOCTL_BLKGETSIZE; extern unsigned IOCTL_BLKRAGET; extern unsigned IOCTL_BLKRASET; extern unsigned IOCTL_BLKROGET; extern unsigned IOCTL_BLKROSET; extern unsigned IOCTL_BLKRRPART; extern unsigned IOCTL_CDROMAUDIOBUFSIZ; extern unsigned IOCTL_CDROMEJECT; extern unsigned IOCTL_CDROMEJECT_SW; extern unsigned IOCTL_CDROMMULTISESSION; extern unsigned IOCTL_CDROMPAUSE; extern unsigned IOCTL_CDROMPLAYMSF; extern unsigned IOCTL_CDROMPLAYTRKIND; extern unsigned IOCTL_CDROMREADAUDIO; extern unsigned IOCTL_CDROMREADCOOKED; extern unsigned IOCTL_CDROMREADMODE1; extern unsigned IOCTL_CDROMREADMODE2; extern unsigned IOCTL_CDROMREADRAW; extern unsigned IOCTL_CDROMREADTOCENTRY; extern unsigned IOCTL_CDROMREADTOCHDR; extern unsigned IOCTL_CDROMRESET; extern unsigned IOCTL_CDROMRESUME; extern unsigned IOCTL_CDROMSEEK; extern unsigned IOCTL_CDROMSTART; extern unsigned IOCTL_CDROMSTOP; extern unsigned IOCTL_CDROMSUBCHNL; extern unsigned IOCTL_CDROMVOLCTRL; extern unsigned IOCTL_CDROMVOLREAD; extern unsigned IOCTL_CDROM_GET_UPC; extern unsigned IOCTL_FDCLRPRM; extern unsigned IOCTL_FDDEFPRM; extern unsigned IOCTL_FDFLUSH; extern unsigned IOCTL_FDFMTBEG; extern unsigned IOCTL_FDFMTEND; extern unsigned IOCTL_FDFMTTRK; extern unsigned IOCTL_FDGETDRVPRM; extern unsigned IOCTL_FDGETDRVSTAT; extern unsigned IOCTL_FDGETDRVTYP; extern unsigned IOCTL_FDGETFDCSTAT; extern unsigned IOCTL_FDGETMAXERRS; extern unsigned IOCTL_FDGETPRM; extern unsigned IOCTL_FDMSGOFF; extern unsigned IOCTL_FDMSGON; extern unsigned IOCTL_FDPOLLDRVSTAT; extern unsigned IOCTL_FDRAWCMD; extern unsigned IOCTL_FDRESET; extern unsigned IOCTL_FDSETDRVPRM; extern unsigned IOCTL_FDSETEMSGTRESH; extern unsigned IOCTL_FDSETMAXERRS; extern unsigned IOCTL_FDSETPRM; extern unsigned IOCTL_FDTWADDLE; extern unsigned IOCTL_FDWERRORCLR; extern unsigned IOCTL_FDWERRORGET; extern unsigned IOCTL_HDIO_DRIVE_CMD; extern unsigned IOCTL_HDIO_GETGEO; extern unsigned IOCTL_HDIO_GET_32BIT; extern unsigned IOCTL_HDIO_GET_DMA; extern unsigned IOCTL_HDIO_GET_IDENTITY; extern unsigned IOCTL_HDIO_GET_KEEPSETTINGS; extern unsigned IOCTL_HDIO_GET_MULTCOUNT; extern unsigned IOCTL_HDIO_GET_NOWERR; extern unsigned IOCTL_HDIO_GET_UNMASKINTR; extern unsigned IOCTL_HDIO_SET_32BIT; extern unsigned IOCTL_HDIO_SET_DMA; extern unsigned IOCTL_HDIO_SET_KEEPSETTINGS; extern unsigned IOCTL_HDIO_SET_MULTCOUNT; extern unsigned IOCTL_HDIO_SET_NOWERR; extern unsigned IOCTL_HDIO_SET_UNMASKINTR; extern unsigned IOCTL_MTIOCPOS; extern unsigned IOCTL_PPPIOCGASYNCMAP; extern unsigned IOCTL_PPPIOCGDEBUG; extern unsigned IOCTL_PPPIOCGFLAGS; extern unsigned IOCTL_PPPIOCGUNIT; extern unsigned IOCTL_PPPIOCGXASYNCMAP; extern unsigned IOCTL_PPPIOCSASYNCMAP; extern unsigned IOCTL_PPPIOCSDEBUG; extern unsigned IOCTL_PPPIOCSFLAGS; extern unsigned IOCTL_PPPIOCSMAXCID; extern unsigned IOCTL_PPPIOCSMRU; extern unsigned IOCTL_PPPIOCSXASYNCMAP; extern unsigned IOCTL_SIOCDARP; extern unsigned IOCTL_SIOCDRARP; extern unsigned IOCTL_SIOCGARP; extern unsigned IOCTL_SIOCGIFENCAP; extern unsigned IOCTL_SIOCGIFHWADDR; extern unsigned IOCTL_SIOCGIFMAP; extern unsigned IOCTL_SIOCGIFMEM; extern unsigned IOCTL_SIOCGIFNAME; extern unsigned IOCTL_SIOCGIFSLAVE; extern unsigned IOCTL_SIOCGRARP; extern unsigned IOCTL_SIOCGSTAMP; extern unsigned IOCTL_SIOCSARP; extern unsigned IOCTL_SIOCSIFENCAP; extern unsigned IOCTL_SIOCSIFHWADDR; extern unsigned IOCTL_SIOCSIFLINK; extern unsigned IOCTL_SIOCSIFMAP; extern unsigned IOCTL_SIOCSIFMEM; extern unsigned IOCTL_SIOCSIFSLAVE; extern unsigned IOCTL_SIOCSRARP; extern unsigned IOCTL_SNDCTL_COPR_HALT; extern unsigned IOCTL_SNDCTL_COPR_LOAD; extern unsigned IOCTL_SNDCTL_COPR_RCODE; extern unsigned IOCTL_SNDCTL_COPR_RCVMSG; extern unsigned IOCTL_SNDCTL_COPR_RDATA; extern unsigned IOCTL_SNDCTL_COPR_RESET; extern unsigned IOCTL_SNDCTL_COPR_RUN; extern unsigned IOCTL_SNDCTL_COPR_SENDMSG; extern unsigned IOCTL_SNDCTL_COPR_WCODE; extern unsigned IOCTL_SNDCTL_COPR_WDATA; extern unsigned IOCTL_TCFLSH; extern unsigned IOCTL_TCGETA; extern unsigned IOCTL_TCGETS; extern unsigned IOCTL_TCSBRK; extern unsigned IOCTL_TCSBRKP; extern unsigned IOCTL_TCSETA; extern unsigned IOCTL_TCSETAF; extern unsigned IOCTL_TCSETAW; extern unsigned IOCTL_TCSETS; extern unsigned IOCTL_TCSETSF; extern unsigned IOCTL_TCSETSW; extern unsigned IOCTL_TCXONC; extern unsigned IOCTL_TIOCGLCKTRMIOS; extern unsigned IOCTL_TIOCGSOFTCAR; extern unsigned IOCTL_TIOCINQ; extern unsigned IOCTL_TIOCLINUX; extern unsigned IOCTL_TIOCSERCONFIG; extern unsigned IOCTL_TIOCSERGETLSR; extern unsigned IOCTL_TIOCSERGWILD; extern unsigned IOCTL_TIOCSERSWILD; extern unsigned IOCTL_TIOCSLCKTRMIOS; extern unsigned IOCTL_TIOCSSOFTCAR; extern unsigned IOCTL_VT_DISALLOCATE; extern unsigned IOCTL_VT_GETSTATE; extern unsigned IOCTL_VT_RESIZE; extern unsigned IOCTL_VT_RESIZEX; extern unsigned IOCTL_VT_SENDSIG; #endif // SANITIZER_LINUX #if SANITIZER_LINUX || SANITIZER_FREEBSD extern unsigned IOCTL_MTIOCGET; extern unsigned IOCTL_MTIOCTOP; extern unsigned IOCTL_SIOCADDRT; extern unsigned IOCTL_SIOCDELRT; extern unsigned IOCTL_SNDCTL_DSP_GETBLKSIZE; extern unsigned IOCTL_SNDCTL_DSP_GETFMTS; extern unsigned IOCTL_SNDCTL_DSP_NONBLOCK; extern unsigned IOCTL_SNDCTL_DSP_POST; extern unsigned IOCTL_SNDCTL_DSP_RESET; extern unsigned IOCTL_SNDCTL_DSP_SETFMT; extern unsigned IOCTL_SNDCTL_DSP_SETFRAGMENT; extern unsigned IOCTL_SNDCTL_DSP_SPEED; extern unsigned IOCTL_SNDCTL_DSP_STEREO; extern unsigned IOCTL_SNDCTL_DSP_SUBDIVIDE; extern unsigned IOCTL_SNDCTL_DSP_SYNC; extern unsigned IOCTL_SNDCTL_FM_4OP_ENABLE; extern unsigned IOCTL_SNDCTL_FM_LOAD_INSTR; extern unsigned IOCTL_SNDCTL_MIDI_INFO; extern unsigned IOCTL_SNDCTL_MIDI_PRETIME; extern unsigned IOCTL_SNDCTL_SEQ_CTRLRATE; extern unsigned IOCTL_SNDCTL_SEQ_GETINCOUNT; extern unsigned IOCTL_SNDCTL_SEQ_GETOUTCOUNT; extern unsigned IOCTL_SNDCTL_SEQ_NRMIDIS; extern unsigned IOCTL_SNDCTL_SEQ_NRSYNTHS; extern unsigned IOCTL_SNDCTL_SEQ_OUTOFBAND; extern unsigned IOCTL_SNDCTL_SEQ_PANIC; extern unsigned IOCTL_SNDCTL_SEQ_PERCMODE; extern unsigned IOCTL_SNDCTL_SEQ_RESET; extern unsigned IOCTL_SNDCTL_SEQ_RESETSAMPLES; extern unsigned IOCTL_SNDCTL_SEQ_SYNC; extern unsigned IOCTL_SNDCTL_SEQ_TESTMIDI; extern unsigned IOCTL_SNDCTL_SEQ_THRESHOLD; extern unsigned IOCTL_SNDCTL_SYNTH_INFO; extern unsigned IOCTL_SNDCTL_SYNTH_MEMAVL; extern unsigned IOCTL_SNDCTL_TMR_CONTINUE; extern unsigned IOCTL_SNDCTL_TMR_METRONOME; extern unsigned IOCTL_SNDCTL_TMR_SELECT; extern unsigned IOCTL_SNDCTL_TMR_SOURCE; extern unsigned IOCTL_SNDCTL_TMR_START; extern unsigned IOCTL_SNDCTL_TMR_STOP; extern unsigned IOCTL_SNDCTL_TMR_TEMPO; extern unsigned IOCTL_SNDCTL_TMR_TIMEBASE; extern unsigned IOCTL_SOUND_MIXER_READ_ALTPCM; extern unsigned IOCTL_SOUND_MIXER_READ_BASS; extern unsigned IOCTL_SOUND_MIXER_READ_CAPS; extern unsigned IOCTL_SOUND_MIXER_READ_CD; extern unsigned IOCTL_SOUND_MIXER_READ_DEVMASK; extern unsigned IOCTL_SOUND_MIXER_READ_ENHANCE; extern unsigned IOCTL_SOUND_MIXER_READ_IGAIN; extern unsigned IOCTL_SOUND_MIXER_READ_IMIX; extern unsigned IOCTL_SOUND_MIXER_READ_LINE1; extern unsigned IOCTL_SOUND_MIXER_READ_LINE2; extern unsigned IOCTL_SOUND_MIXER_READ_LINE3; extern unsigned IOCTL_SOUND_MIXER_READ_LINE; extern unsigned IOCTL_SOUND_MIXER_READ_LOUD; extern unsigned IOCTL_SOUND_MIXER_READ_MIC; extern unsigned IOCTL_SOUND_MIXER_READ_MUTE; extern unsigned IOCTL_SOUND_MIXER_READ_OGAIN; extern unsigned IOCTL_SOUND_MIXER_READ_PCM; extern unsigned IOCTL_SOUND_MIXER_READ_RECLEV; extern unsigned IOCTL_SOUND_MIXER_READ_RECMASK; extern unsigned IOCTL_SOUND_MIXER_READ_RECSRC; extern unsigned IOCTL_SOUND_MIXER_READ_SPEAKER; extern unsigned IOCTL_SOUND_MIXER_READ_STEREODEVS; extern unsigned IOCTL_SOUND_MIXER_READ_SYNTH; extern unsigned IOCTL_SOUND_MIXER_READ_TREBLE; extern unsigned IOCTL_SOUND_MIXER_READ_VOLUME; extern unsigned IOCTL_SOUND_MIXER_WRITE_ALTPCM; extern unsigned IOCTL_SOUND_MIXER_WRITE_BASS; extern unsigned IOCTL_SOUND_MIXER_WRITE_CD; extern unsigned IOCTL_SOUND_MIXER_WRITE_ENHANCE; extern unsigned IOCTL_SOUND_MIXER_WRITE_IGAIN; extern unsigned IOCTL_SOUND_MIXER_WRITE_IMIX; extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE1; extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE2; extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE3; extern unsigned IOCTL_SOUND_MIXER_WRITE_LINE; extern unsigned IOCTL_SOUND_MIXER_WRITE_LOUD; extern unsigned IOCTL_SOUND_MIXER_WRITE_MIC; extern unsigned IOCTL_SOUND_MIXER_WRITE_MUTE; extern unsigned IOCTL_SOUND_MIXER_WRITE_OGAIN; extern unsigned IOCTL_SOUND_MIXER_WRITE_PCM; extern unsigned IOCTL_SOUND_MIXER_WRITE_RECLEV; extern unsigned IOCTL_SOUND_MIXER_WRITE_RECSRC; extern unsigned IOCTL_SOUND_MIXER_WRITE_SPEAKER; extern unsigned IOCTL_SOUND_MIXER_WRITE_SYNTH; extern unsigned IOCTL_SOUND_MIXER_WRITE_TREBLE; extern unsigned IOCTL_SOUND_MIXER_WRITE_VOLUME; extern unsigned IOCTL_SOUND_PCM_READ_BITS; extern unsigned IOCTL_SOUND_PCM_READ_CHANNELS; extern unsigned IOCTL_SOUND_PCM_READ_FILTER; extern unsigned IOCTL_SOUND_PCM_READ_RATE; extern unsigned IOCTL_SOUND_PCM_WRITE_CHANNELS; extern unsigned IOCTL_SOUND_PCM_WRITE_FILTER; extern unsigned IOCTL_VT_ACTIVATE; extern unsigned IOCTL_VT_GETMODE; extern unsigned IOCTL_VT_OPENQRY; extern unsigned IOCTL_VT_RELDISP; extern unsigned IOCTL_VT_SETMODE; extern unsigned IOCTL_VT_WAITACTIVE; #endif // SANITIZER_LINUX || SANITIZER_FREEBSD #if SANITIZER_LINUX && !SANITIZER_ANDROID extern unsigned IOCTL_CYGETDEFTHRESH; extern unsigned IOCTL_CYGETDEFTIMEOUT; extern unsigned IOCTL_CYGETMON; extern unsigned IOCTL_CYGETTHRESH; extern unsigned IOCTL_CYGETTIMEOUT; extern unsigned IOCTL_CYSETDEFTHRESH; extern unsigned IOCTL_CYSETDEFTIMEOUT; extern unsigned IOCTL_CYSETTHRESH; extern unsigned IOCTL_CYSETTIMEOUT; extern unsigned IOCTL_EQL_EMANCIPATE; extern unsigned IOCTL_EQL_ENSLAVE; extern unsigned IOCTL_EQL_GETMASTRCFG; extern unsigned IOCTL_EQL_GETSLAVECFG; extern unsigned IOCTL_EQL_SETMASTRCFG; extern unsigned IOCTL_EQL_SETSLAVECFG; extern unsigned IOCTL_EVIOCGKEYCODE_V2; extern unsigned IOCTL_EVIOCGPROP; extern unsigned IOCTL_EVIOCSKEYCODE_V2; extern unsigned IOCTL_FS_IOC_GETFLAGS; extern unsigned IOCTL_FS_IOC_GETVERSION; extern unsigned IOCTL_FS_IOC_SETFLAGS; extern unsigned IOCTL_FS_IOC_SETVERSION; extern unsigned IOCTL_GIO_CMAP; extern unsigned IOCTL_GIO_FONT; extern unsigned IOCTL_GIO_UNIMAP; extern unsigned IOCTL_GIO_UNISCRNMAP; extern unsigned IOCTL_KDADDIO; extern unsigned IOCTL_KDDELIO; extern unsigned IOCTL_KDGETKEYCODE; extern unsigned IOCTL_KDGKBDIACR; extern unsigned IOCTL_KDGKBENT; extern unsigned IOCTL_KDGKBLED; extern unsigned IOCTL_KDGKBMETA; extern unsigned IOCTL_KDGKBSENT; extern unsigned IOCTL_KDMAPDISP; extern unsigned IOCTL_KDSETKEYCODE; extern unsigned IOCTL_KDSIGACCEPT; extern unsigned IOCTL_KDSKBDIACR; extern unsigned IOCTL_KDSKBENT; extern unsigned IOCTL_KDSKBLED; extern unsigned IOCTL_KDSKBMETA; extern unsigned IOCTL_KDSKBSENT; extern unsigned IOCTL_KDUNMAPDISP; extern unsigned IOCTL_LPABORT; extern unsigned IOCTL_LPABORTOPEN; extern unsigned IOCTL_LPCAREFUL; extern unsigned IOCTL_LPCHAR; extern unsigned IOCTL_LPGETIRQ; extern unsigned IOCTL_LPGETSTATUS; extern unsigned IOCTL_LPRESET; extern unsigned IOCTL_LPSETIRQ; extern unsigned IOCTL_LPTIME; extern unsigned IOCTL_LPWAIT; extern unsigned IOCTL_MTIOCGETCONFIG; extern unsigned IOCTL_MTIOCSETCONFIG; extern unsigned IOCTL_PIO_CMAP; extern unsigned IOCTL_PIO_FONT; extern unsigned IOCTL_PIO_UNIMAP; extern unsigned IOCTL_PIO_UNIMAPCLR; extern unsigned IOCTL_PIO_UNISCRNMAP; extern unsigned IOCTL_SCSI_IOCTL_GET_IDLUN; extern unsigned IOCTL_SCSI_IOCTL_PROBE_HOST; extern unsigned IOCTL_SCSI_IOCTL_TAGGED_DISABLE; extern unsigned IOCTL_SCSI_IOCTL_TAGGED_ENABLE; extern unsigned IOCTL_SIOCAIPXITFCRT; extern unsigned IOCTL_SIOCAIPXPRISLT; extern unsigned IOCTL_SIOCAX25ADDUID; extern unsigned IOCTL_SIOCAX25DELUID; extern unsigned IOCTL_SIOCAX25GETPARMS; extern unsigned IOCTL_SIOCAX25GETUID; extern unsigned IOCTL_SIOCAX25NOUID; extern unsigned IOCTL_SIOCAX25SETPARMS; extern unsigned IOCTL_SIOCDEVPLIP; extern unsigned IOCTL_SIOCIPXCFGDATA; extern unsigned IOCTL_SIOCNRDECOBS; extern unsigned IOCTL_SIOCNRGETPARMS; extern unsigned IOCTL_SIOCNRRTCTL; extern unsigned IOCTL_SIOCNRSETPARMS; extern unsigned IOCTL_SNDCTL_DSP_GETISPACE; extern unsigned IOCTL_SNDCTL_DSP_GETOSPACE; extern unsigned IOCTL_TIOCGSERIAL; extern unsigned IOCTL_TIOCSERGETMULTI; extern unsigned IOCTL_TIOCSERSETMULTI; extern unsigned IOCTL_TIOCSSERIAL; #endif // SANITIZER_LINUX && !SANITIZER_ANDROID #if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID extern unsigned IOCTL_GIO_SCRNMAP; extern unsigned IOCTL_KDDISABIO; extern unsigned IOCTL_KDENABIO; extern unsigned IOCTL_KDGETLED; extern unsigned IOCTL_KDGETMODE; extern unsigned IOCTL_KDGKBMODE; extern unsigned IOCTL_KDGKBTYPE; extern unsigned IOCTL_KDMKTONE; extern unsigned IOCTL_KDSETLED; extern unsigned IOCTL_KDSETMODE; extern unsigned IOCTL_KDSKBMODE; extern unsigned IOCTL_KIOCSOUND; extern unsigned IOCTL_PIO_SCRNMAP; #endif extern const int errno_EINVAL; extern const int errno_EOWNERDEAD; extern const int si_SEGV_MAPERR; extern const int si_SEGV_ACCERR; } // namespace __sanitizer #define CHECK_TYPE_SIZE(TYPE) \ COMPILER_CHECK(sizeof(__sanitizer_##TYPE) == sizeof(TYPE)) #define CHECK_SIZE_AND_OFFSET(CLASS, MEMBER) \ COMPILER_CHECK(sizeof(((__sanitizer_##CLASS *) NULL)->MEMBER) == \ sizeof(((CLASS *) NULL)->MEMBER)); \ COMPILER_CHECK(offsetof(__sanitizer_##CLASS, MEMBER) == \ offsetof(CLASS, MEMBER)) // For sigaction, which is a function and struct at the same time, // and thus requires explicit "struct" in sizeof() expression. #define CHECK_STRUCT_SIZE_AND_OFFSET(CLASS, MEMBER) \ COMPILER_CHECK(sizeof(((struct __sanitizer_##CLASS *) NULL)->MEMBER) == \ sizeof(((struct CLASS *) NULL)->MEMBER)); \ COMPILER_CHECK(offsetof(struct __sanitizer_##CLASS, MEMBER) == \ offsetof(struct CLASS, MEMBER)) #endif Index: projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h (revision 312198) @@ -1,210 +1,224 @@ //===-- sanitizer_quarantine.h ----------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // Memory quarantine for AddressSanitizer and potentially other tools. // Quarantine caches some specified amount of memory in per-thread caches, // then evicts to global FIFO queue. When the queue reaches specified threshold, // oldest memory is recycled. // //===----------------------------------------------------------------------===// #ifndef SANITIZER_QUARANTINE_H #define SANITIZER_QUARANTINE_H #include "sanitizer_internal_defs.h" #include "sanitizer_mutex.h" #include "sanitizer_list.h" namespace __sanitizer { template class QuarantineCache; struct QuarantineBatch { static const uptr kSize = 1021; QuarantineBatch *next; uptr size; uptr count; void *batch[kSize]; }; COMPILER_CHECK(sizeof(QuarantineBatch) <= (1 << 13)); // 8Kb. // The callback interface is: // void Callback::Recycle(Node *ptr); // void *cb.Allocate(uptr size); // void cb.Deallocate(void *ptr); template class Quarantine { public: typedef QuarantineCache Cache; explicit Quarantine(LinkerInitialized) : cache_(LINKER_INITIALIZED) { } void Init(uptr size, uptr cache_size) { - atomic_store(&max_size_, size, memory_order_release); + // Thread local quarantine size can be zero only when global quarantine size + // is zero (it allows us to perform just one atomic read per Put() call). + CHECK((size == 0 && cache_size == 0) || cache_size != 0); + + atomic_store(&max_size_, size, memory_order_relaxed); atomic_store(&min_size_, size / 10 * 9, - memory_order_release); // 90% of max size. - max_cache_size_ = cache_size; + memory_order_relaxed); // 90% of max size. + atomic_store(&max_cache_size_, cache_size, memory_order_relaxed); } - uptr GetSize() const { return atomic_load(&max_size_, memory_order_acquire); } - uptr GetCacheSize() const { return max_cache_size_; } + uptr GetSize() const { return atomic_load(&max_size_, memory_order_relaxed); } + uptr GetCacheSize() const { + return atomic_load(&max_cache_size_, memory_order_relaxed); + } void Put(Cache *c, Callback cb, Node *ptr, uptr size) { - c->Enqueue(cb, ptr, size); - if (c->Size() > max_cache_size_) + uptr cache_size = GetCacheSize(); + if (cache_size) { + c->Enqueue(cb, ptr, size); + } else { + // cache_size == 0 only when size == 0 (see Init). + cb.Recycle(ptr); + } + // Check cache size anyway to accommodate for runtime cache_size change. + if (c->Size() > cache_size) Drain(c, cb); } void NOINLINE Drain(Cache *c, Callback cb) { { SpinMutexLock l(&cache_mutex_); cache_.Transfer(c); } if (cache_.Size() > GetSize() && recycle_mutex_.TryLock()) Recycle(cb); } void PrintStats() const { // It assumes that the world is stopped, just as the allocator's PrintStats. cache_.PrintStats(); } private: // Read-only data. char pad0_[kCacheLineSize]; atomic_uintptr_t max_size_; atomic_uintptr_t min_size_; - uptr max_cache_size_; + atomic_uintptr_t max_cache_size_; char pad1_[kCacheLineSize]; SpinMutex cache_mutex_; SpinMutex recycle_mutex_; Cache cache_; char pad2_[kCacheLineSize]; void NOINLINE Recycle(Callback cb) { Cache tmp; - uptr min_size = atomic_load(&min_size_, memory_order_acquire); + uptr min_size = atomic_load(&min_size_, memory_order_relaxed); { SpinMutexLock l(&cache_mutex_); while (cache_.Size() > min_size) { QuarantineBatch *b = cache_.DequeueBatch(); tmp.EnqueueBatch(b); } } recycle_mutex_.Unlock(); DoRecycle(&tmp, cb); } void NOINLINE DoRecycle(Cache *c, Callback cb) { while (QuarantineBatch *b = c->DequeueBatch()) { const uptr kPrefetch = 16; CHECK(kPrefetch <= ARRAY_SIZE(b->batch)); for (uptr i = 0; i < kPrefetch; i++) PREFETCH(b->batch[i]); for (uptr i = 0, count = b->count; i < count; i++) { if (i + kPrefetch < count) PREFETCH(b->batch[i + kPrefetch]); cb.Recycle((Node*)b->batch[i]); } cb.Deallocate(b); } } }; // Per-thread cache of memory blocks. template class QuarantineCache { public: explicit QuarantineCache(LinkerInitialized) { } QuarantineCache() : size_() { list_.clear(); } uptr Size() const { return atomic_load(&size_, memory_order_relaxed); } void Enqueue(Callback cb, void *ptr, uptr size) { if (list_.empty() || list_.back()->count == QuarantineBatch::kSize) { AllocBatch(cb); size += sizeof(QuarantineBatch); // Count the batch in Quarantine size. } QuarantineBatch *b = list_.back(); CHECK(b); b->batch[b->count++] = ptr; b->size += size; SizeAdd(size); } void Transfer(QuarantineCache *c) { list_.append_back(&c->list_); SizeAdd(c->Size()); atomic_store(&c->size_, 0, memory_order_relaxed); } void EnqueueBatch(QuarantineBatch *b) { list_.push_back(b); SizeAdd(b->size); } QuarantineBatch *DequeueBatch() { if (list_.empty()) return nullptr; QuarantineBatch *b = list_.front(); list_.pop_front(); SizeSub(b->size); return b; } void PrintStats() const { uptr batch_count = 0; uptr total_quarantine_bytes = 0; uptr total_quarantine_chunks = 0; for (List::ConstIterator it = list_.begin(); it != list_.end(); ++it) { batch_count++; total_quarantine_bytes += (*it).size; total_quarantine_chunks += (*it).count; } Printf("Global quarantine stats: batches: %zd; bytes: %zd; chunks: %zd " "(capacity: %zd chunks)\n", batch_count, total_quarantine_bytes, total_quarantine_chunks, batch_count * QuarantineBatch::kSize); } private: typedef IntrusiveList List; List list_; atomic_uintptr_t size_; void SizeAdd(uptr add) { atomic_store(&size_, Size() + add, memory_order_relaxed); } void SizeSub(uptr sub) { atomic_store(&size_, Size() - sub, memory_order_relaxed); } NOINLINE QuarantineBatch* AllocBatch(Callback cb) { QuarantineBatch *b = (QuarantineBatch *)cb.Allocate(sizeof(*b)); CHECK(b); b->count = 0; b->size = 0; list_.push_back(b); return b; } }; + } // namespace __sanitizer #endif // SANITIZER_QUARANTINE_H Index: projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_allocator.cpp (revision 312198) @@ -1,720 +1,693 @@ //===-- scudo_allocator.cpp -------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// /// /// Scudo Hardened Allocator implementation. /// It uses the sanitizer_common allocator as a base and aims at mitigating /// heap corruption vulnerabilities. It provides a checksum-guarded chunk /// header, a delayed free list, and additional sanity checks. /// //===----------------------------------------------------------------------===// #include "scudo_allocator.h" +#include "scudo_crc32.h" #include "scudo_utils.h" #include "sanitizer_common/sanitizer_allocator_interface.h" #include "sanitizer_common/sanitizer_quarantine.h" #include #include #include -// Hardware CRC32 is supported at compilation via the following: -// - for i386 & x86_64: -msse4.2 -// - for ARM & AArch64: -march=armv8-a+crc -// An additional check must be performed at runtime as well to make sure the -// emitted instructions are valid on the target host. -#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) -# ifdef __SSE4_2__ -# include -# define HW_CRC32 FIRST_32_SECOND_64(_mm_crc32_u32, _mm_crc32_u64) -# endif -# ifdef __ARM_FEATURE_CRC32 -# include -# define HW_CRC32 FIRST_32_SECOND_64(__crc32cw, __crc32cd) -# endif -#endif - namespace __scudo { #if SANITIZER_CAN_USE_ALLOCATOR64 const uptr AllocatorSpace = ~0ULL; const uptr AllocatorSize = 0x40000000000ULL; typedef DefaultSizeClassMap SizeClassMap; struct AP { static const uptr kSpaceBeg = AllocatorSpace; static const uptr kSpaceSize = AllocatorSize; static const uptr kMetadataSize = 0; typedef __scudo::SizeClassMap SizeClassMap; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = SizeClassAllocator64FlagMasks::kRandomShuffleChunks; }; typedef SizeClassAllocator64 PrimaryAllocator; #else // Currently, the 32-bit Sanitizer allocator has not yet benefited from all the // security improvements brought to the 64-bit one. This makes the 32-bit // version of Scudo slightly less toughened. static const uptr RegionSizeLog = 20; static const uptr NumRegions = SANITIZER_MMAP_RANGE_SIZE >> RegionSizeLog; # if SANITIZER_WORDSIZE == 32 typedef FlatByteMap ByteMap; # elif SANITIZER_WORDSIZE == 64 typedef TwoLevelByteMap<(NumRegions >> 12), 1 << 12> ByteMap; # endif // SANITIZER_WORDSIZE typedef DefaultSizeClassMap SizeClassMap; typedef SizeClassAllocator32<0, SANITIZER_MMAP_RANGE_SIZE, 0, SizeClassMap, RegionSizeLog, ByteMap> PrimaryAllocator; #endif // SANITIZER_CAN_USE_ALLOCATOR64 typedef SizeClassAllocatorLocalCache AllocatorCache; typedef ScudoLargeMmapAllocator SecondaryAllocator; typedef CombinedAllocator ScudoAllocator; static ScudoAllocator &getAllocator(); static thread_local Xorshift128Plus Prng; // Global static cookie, initialized at start-up. static uptr Cookie; -enum : u8 { - CRC32Software = 0, - CRC32Hardware = 1, -}; // We default to software CRC32 if the alternatives are not supported, either // at compilation or at runtime. static atomic_uint8_t HashAlgorithm = { CRC32Software }; // Helper function that will compute the chunk checksum, being passed all the // the needed information as uptrs. It will opt for the hardware version of // the checksumming function if available. INLINE u32 hashUptrs(uptr Pointer, uptr *Array, uptr ArraySize, u8 HashType) { u32 Crc; -#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) - if (HashType == CRC32Hardware) { - Crc = HW_CRC32(Cookie, Pointer); - for (uptr i = 0; i < ArraySize; i++) - Crc = HW_CRC32(Crc, Array[i]); - return Crc; - } -#endif - Crc = computeCRC32(Cookie, Pointer); + Crc = computeCRC32(Cookie, Pointer, HashType); for (uptr i = 0; i < ArraySize; i++) - Crc = computeCRC32(Crc, Array[i]); + Crc = computeCRC32(Crc, Array[i], HashType); return Crc; } struct ScudoChunk : UnpackedHeader { // We can't use the offset member of the chunk itself, as we would double // fetch it without any warranty that it wouldn't have been tampered. To // prevent this, we work with a local copy of the header. void *getAllocBeg(UnpackedHeader *Header) { return reinterpret_cast( reinterpret_cast(this) - (Header->Offset << MinAlignmentLog)); } // Returns the usable size for a chunk, meaning the amount of bytes from the // beginning of the user data to the end of the backend allocated chunk. uptr getUsableSize(UnpackedHeader *Header) { uptr Size = getAllocator().GetActuallyAllocatedSize(getAllocBeg(Header)); if (Size == 0) return Size; return Size - AlignedChunkHeaderSize - (Header->Offset << MinAlignmentLog); } // Compute the checksum of the Chunk pointer and its ChunkHeader. u16 computeChecksum(UnpackedHeader *Header) const { UnpackedHeader ZeroChecksumHeader = *Header; ZeroChecksumHeader.Checksum = 0; uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)]; memcpy(&HeaderHolder, &ZeroChecksumHeader, sizeof(HeaderHolder)); u32 Hash = hashUptrs(reinterpret_cast(this), HeaderHolder, ARRAY_SIZE(HeaderHolder), atomic_load_relaxed(&HashAlgorithm)); return static_cast(Hash); } // Checks the validity of a chunk by verifying its checksum. bool isValid() { UnpackedHeader NewUnpackedHeader; const AtomicPackedHeader *AtomicHeader = reinterpret_cast(this); PackedHeader NewPackedHeader = AtomicHeader->load(std::memory_order_relaxed); NewUnpackedHeader = bit_cast(NewPackedHeader); return (NewUnpackedHeader.Checksum == computeChecksum(&NewUnpackedHeader)); } // Loads and unpacks the header, verifying the checksum in the process. void loadHeader(UnpackedHeader *NewUnpackedHeader) const { const AtomicPackedHeader *AtomicHeader = reinterpret_cast(this); PackedHeader NewPackedHeader = AtomicHeader->load(std::memory_order_relaxed); *NewUnpackedHeader = bit_cast(NewPackedHeader); if (NewUnpackedHeader->Checksum != computeChecksum(NewUnpackedHeader)) { dieWithMessage("ERROR: corrupted chunk header at address %p\n", this); } } // Packs and stores the header, computing the checksum in the process. void storeHeader(UnpackedHeader *NewUnpackedHeader) { NewUnpackedHeader->Checksum = computeChecksum(NewUnpackedHeader); PackedHeader NewPackedHeader = bit_cast(*NewUnpackedHeader); AtomicPackedHeader *AtomicHeader = reinterpret_cast(this); AtomicHeader->store(NewPackedHeader, std::memory_order_relaxed); } // Packs and stores the header, computing the checksum in the process. We // compare the current header with the expected provided one to ensure that // we are not being raced by a corruption occurring in another thread. void compareExchangeHeader(UnpackedHeader *NewUnpackedHeader, UnpackedHeader *OldUnpackedHeader) { NewUnpackedHeader->Checksum = computeChecksum(NewUnpackedHeader); PackedHeader NewPackedHeader = bit_cast(*NewUnpackedHeader); PackedHeader OldPackedHeader = bit_cast(*OldUnpackedHeader); AtomicPackedHeader *AtomicHeader = reinterpret_cast(this); if (!AtomicHeader->compare_exchange_strong(OldPackedHeader, NewPackedHeader, std::memory_order_relaxed, std::memory_order_relaxed)) { dieWithMessage("ERROR: race on chunk header at address %p\n", this); } } }; static bool ScudoInitIsRunning = false; static pthread_once_t GlobalInited = PTHREAD_ONCE_INIT; static pthread_key_t PThreadKey; static thread_local bool ThreadInited = false; static thread_local bool ThreadTornDown = false; static thread_local AllocatorCache Cache; static void teardownThread(void *p) { uptr v = reinterpret_cast(p); // The glibc POSIX thread-local-storage deallocation routine calls user // provided destructors in a loop of PTHREAD_DESTRUCTOR_ITERATIONS. // We want to be called last since other destructors might call free and the // like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the // quarantine and swallowing the cache. if (v < PTHREAD_DESTRUCTOR_ITERATIONS) { pthread_setspecific(PThreadKey, reinterpret_cast(v + 1)); return; } drainQuarantine(); getAllocator().DestroyCache(&Cache); ThreadTornDown = true; } static void initInternal() { SanitizerToolName = "Scudo"; CHECK(!ScudoInitIsRunning && "Scudo init calls itself!"); ScudoInitIsRunning = true; // Check is SSE4.2 is supported, if so, opt for the CRC32 hardware version. if (testCPUFeature(CRC32CPUFeature)) { atomic_store_relaxed(&HashAlgorithm, CRC32Hardware); } initFlags(); AllocatorOptions Options; Options.setFrom(getFlags(), common_flags()); initAllocator(Options); MaybeStartBackgroudThread(); ScudoInitIsRunning = false; } static void initGlobal() { pthread_key_create(&PThreadKey, teardownThread); initInternal(); } static void NOINLINE initThread() { pthread_once(&GlobalInited, initGlobal); pthread_setspecific(PThreadKey, reinterpret_cast(1)); getAllocator().InitCache(&Cache); ThreadInited = true; } struct QuarantineCallback { explicit QuarantineCallback(AllocatorCache *Cache) : Cache_(Cache) {} // Chunk recycling function, returns a quarantined chunk to the backend. void Recycle(ScudoChunk *Chunk) { UnpackedHeader Header; Chunk->loadHeader(&Header); if (Header.State != ChunkQuarantine) { dieWithMessage("ERROR: invalid chunk state when recycling address %p\n", Chunk); } void *Ptr = Chunk->getAllocBeg(&Header); getAllocator().Deallocate(Cache_, Ptr); } /// Internal quarantine allocation and deallocation functions. void *Allocate(uptr Size) { // The internal quarantine memory cannot be protected by us. But the only // structures allocated are QuarantineBatch, that are 8KB for x64. So we // will use mmap for those, and given that Deallocate doesn't pass a size // in, we enforce the size of the allocation to be sizeof(QuarantineBatch). // TODO(kostyak): switching to mmap impacts greatly performances, we have // to find another solution // CHECK_EQ(Size, sizeof(QuarantineBatch)); // return MmapOrDie(Size, "QuarantineBatch"); return getAllocator().Allocate(Cache_, Size, 1, false); } void Deallocate(void *Ptr) { // UnmapOrDie(Ptr, sizeof(QuarantineBatch)); getAllocator().Deallocate(Cache_, Ptr); } AllocatorCache *Cache_; }; typedef Quarantine ScudoQuarantine; typedef ScudoQuarantine::Cache QuarantineCache; static thread_local QuarantineCache ThreadQuarantineCache; void AllocatorOptions::setFrom(const Flags *f, const CommonFlags *cf) { MayReturnNull = cf->allocator_may_return_null; ReleaseToOSIntervalMs = cf->allocator_release_to_os_interval_ms; QuarantineSizeMb = f->QuarantineSizeMb; ThreadLocalQuarantineSizeKb = f->ThreadLocalQuarantineSizeKb; DeallocationTypeMismatch = f->DeallocationTypeMismatch; DeleteSizeMismatch = f->DeleteSizeMismatch; ZeroContents = f->ZeroContents; } void AllocatorOptions::copyTo(Flags *f, CommonFlags *cf) const { cf->allocator_may_return_null = MayReturnNull; cf->allocator_release_to_os_interval_ms = ReleaseToOSIntervalMs; f->QuarantineSizeMb = QuarantineSizeMb; f->ThreadLocalQuarantineSizeKb = ThreadLocalQuarantineSizeKb; f->DeallocationTypeMismatch = DeallocationTypeMismatch; f->DeleteSizeMismatch = DeleteSizeMismatch; f->ZeroContents = ZeroContents; } struct Allocator { static const uptr MaxAllowedMallocSize = FIRST_32_SECOND_64(2UL << 30, 1ULL << 40); ScudoAllocator BackendAllocator; ScudoQuarantine AllocatorQuarantine; // The fallback caches are used when the thread local caches have been // 'detroyed' on thread tear-down. They are protected by a Mutex as they can // be accessed by different threads. StaticSpinMutex FallbackMutex; AllocatorCache FallbackAllocatorCache; QuarantineCache FallbackQuarantineCache; bool DeallocationTypeMismatch; bool ZeroContents; bool DeleteSizeMismatch; explicit Allocator(LinkerInitialized) : AllocatorQuarantine(LINKER_INITIALIZED), FallbackQuarantineCache(LINKER_INITIALIZED) {} void init(const AllocatorOptions &Options) { // Verify that the header offset field can hold the maximum offset. In the // case of the Secondary allocator, it takes care of alignment and the // offset will always be 0. In the case of the Primary, the worst case // scenario happens in the last size class, when the backend allocation // would already be aligned on the requested alignment, which would happen // to be the maximum alignment that would fit in that size class. As a // result, the maximum offset will be at most the maximum alignment for the // last size class minus the header size, in multiples of MinAlignment. UnpackedHeader Header = {}; uptr MaxPrimaryAlignment = 1 << MostSignificantSetBitIndex( SizeClassMap::kMaxSize - MinAlignment); uptr MaxOffset = (MaxPrimaryAlignment - AlignedChunkHeaderSize) >> MinAlignmentLog; Header.Offset = MaxOffset; if (Header.Offset != MaxOffset) { dieWithMessage("ERROR: the maximum possible offset doesn't fit in the " "header\n"); } // Verify that we can fit the maximum amount of unused bytes in the header. // Given that the Secondary fits the allocation to a page, the worst case // scenario happens in the Primary. It will depend on the second to last // and last class sizes, as well as the dynamic base for the Primary. The // following is an over-approximation that works for our needs. uptr MaxUnusedBytes = SizeClassMap::kMaxSize - 1 - AlignedChunkHeaderSize; Header.UnusedBytes = MaxUnusedBytes; if (Header.UnusedBytes != MaxUnusedBytes) { dieWithMessage("ERROR: the maximum possible unused bytes doesn't fit in " "the header\n"); } DeallocationTypeMismatch = Options.DeallocationTypeMismatch; DeleteSizeMismatch = Options.DeleteSizeMismatch; ZeroContents = Options.ZeroContents; BackendAllocator.Init(Options.MayReturnNull, Options.ReleaseToOSIntervalMs); AllocatorQuarantine.Init( static_cast(Options.QuarantineSizeMb) << 20, static_cast(Options.ThreadLocalQuarantineSizeKb) << 10); BackendAllocator.InitCache(&FallbackAllocatorCache); Cookie = Prng.Next(); } // Helper function that checks for a valid Scudo chunk. bool isValidPointer(const void *UserPtr) { uptr ChunkBeg = reinterpret_cast(UserPtr); if (!IsAligned(ChunkBeg, MinAlignment)) { return false; } ScudoChunk *Chunk = reinterpret_cast(ChunkBeg - AlignedChunkHeaderSize); return Chunk->isValid(); } // Allocates a chunk. void *allocate(uptr Size, uptr Alignment, AllocType Type) { if (UNLIKELY(!ThreadInited)) initThread(); if (!IsPowerOfTwo(Alignment)) { dieWithMessage("ERROR: alignment is not a power of 2\n"); } if (Alignment > MaxAlignment) return BackendAllocator.ReturnNullOrDieOnBadRequest(); if (Alignment < MinAlignment) Alignment = MinAlignment; if (Size == 0) Size = 1; if (Size >= MaxAllowedMallocSize) return BackendAllocator.ReturnNullOrDieOnBadRequest(); uptr NeededSize = RoundUpTo(Size, MinAlignment) + AlignedChunkHeaderSize; if (Alignment > MinAlignment) NeededSize += Alignment; if (NeededSize >= MaxAllowedMallocSize) return BackendAllocator.ReturnNullOrDieOnBadRequest(); // Primary backed and Secondary backed allocations have a different // treatment. We deal with alignment requirements of Primary serviced // allocations here, but the Secondary will take care of its own alignment // needs, which means we also have to work around some limitations of the // combined allocator to accommodate the situation. bool FromPrimary = PrimaryAllocator::CanAllocate(NeededSize, MinAlignment); void *Ptr; if (LIKELY(!ThreadTornDown)) { Ptr = BackendAllocator.Allocate(&Cache, NeededSize, FromPrimary ? MinAlignment : Alignment); } else { SpinMutexLock l(&FallbackMutex); Ptr = BackendAllocator.Allocate(&FallbackAllocatorCache, NeededSize, FromPrimary ? MinAlignment : Alignment); } if (!Ptr) return BackendAllocator.ReturnNullOrDieOnOOM(); uptr AllocBeg = reinterpret_cast(Ptr); // If the allocation was serviced by the secondary, the returned pointer // accounts for ChunkHeaderSize to pass the alignment check of the combined // allocator. Adjust it here. if (!FromPrimary) { AllocBeg -= AlignedChunkHeaderSize; if (Alignment > MinAlignment) NeededSize -= Alignment; } uptr ActuallyAllocatedSize = BackendAllocator.GetActuallyAllocatedSize( reinterpret_cast(AllocBeg)); // If requested, we will zero out the entire contents of the returned chunk. if (ZeroContents && FromPrimary) memset(Ptr, 0, ActuallyAllocatedSize); uptr ChunkBeg = AllocBeg + AlignedChunkHeaderSize; if (!IsAligned(ChunkBeg, Alignment)) ChunkBeg = RoundUpTo(ChunkBeg, Alignment); CHECK_LE(ChunkBeg + Size, AllocBeg + NeededSize); ScudoChunk *Chunk = reinterpret_cast(ChunkBeg - AlignedChunkHeaderSize); UnpackedHeader Header = {}; Header.State = ChunkAllocated; uptr Offset = ChunkBeg - AlignedChunkHeaderSize - AllocBeg; Header.Offset = Offset >> MinAlignmentLog; Header.AllocType = Type; Header.UnusedBytes = ActuallyAllocatedSize - Offset - AlignedChunkHeaderSize - Size; Header.Salt = static_cast(Prng.Next()); Chunk->storeHeader(&Header); void *UserPtr = reinterpret_cast(ChunkBeg); // TODO(kostyak): hooks sound like a terrible idea security wise but might // be needed for things to work properly? // if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(UserPtr, Size); return UserPtr; } // Deallocates a Chunk, which means adding it to the delayed free list (or // Quarantine). void deallocate(void *UserPtr, uptr DeleteSize, AllocType Type) { if (UNLIKELY(!ThreadInited)) initThread(); // TODO(kostyak): see hook comment above // if (&__sanitizer_free_hook) __sanitizer_free_hook(UserPtr); if (!UserPtr) return; uptr ChunkBeg = reinterpret_cast(UserPtr); if (!IsAligned(ChunkBeg, MinAlignment)) { dieWithMessage("ERROR: attempted to deallocate a chunk not properly " "aligned at address %p\n", UserPtr); } ScudoChunk *Chunk = reinterpret_cast(ChunkBeg - AlignedChunkHeaderSize); UnpackedHeader OldHeader; Chunk->loadHeader(&OldHeader); if (OldHeader.State != ChunkAllocated) { dieWithMessage("ERROR: invalid chunk state when deallocating address " "%p\n", UserPtr); } uptr UsableSize = Chunk->getUsableSize(&OldHeader); UnpackedHeader NewHeader = OldHeader; NewHeader.State = ChunkQuarantine; Chunk->compareExchangeHeader(&NewHeader, &OldHeader); if (DeallocationTypeMismatch) { // The deallocation type has to match the allocation one. if (NewHeader.AllocType != Type) { // With the exception of memalign'd Chunks, that can be still be free'd. if (NewHeader.AllocType != FromMemalign || Type != FromMalloc) { dieWithMessage("ERROR: allocation type mismatch on address %p\n", Chunk); } } } uptr Size = UsableSize - OldHeader.UnusedBytes; if (DeleteSizeMismatch) { if (DeleteSize && DeleteSize != Size) { dieWithMessage("ERROR: invalid sized delete on chunk at address %p\n", Chunk); } } if (LIKELY(!ThreadTornDown)) { AllocatorQuarantine.Put(&ThreadQuarantineCache, QuarantineCallback(&Cache), Chunk, UsableSize); } else { SpinMutexLock l(&FallbackMutex); AllocatorQuarantine.Put(&FallbackQuarantineCache, QuarantineCallback(&FallbackAllocatorCache), Chunk, UsableSize); } } // Reallocates a chunk. We can save on a new allocation if the new requested // size still fits in the chunk. void *reallocate(void *OldPtr, uptr NewSize) { if (UNLIKELY(!ThreadInited)) initThread(); uptr ChunkBeg = reinterpret_cast(OldPtr); ScudoChunk *Chunk = reinterpret_cast(ChunkBeg - AlignedChunkHeaderSize); UnpackedHeader OldHeader; Chunk->loadHeader(&OldHeader); if (OldHeader.State != ChunkAllocated) { dieWithMessage("ERROR: invalid chunk state when reallocating address " "%p\n", OldPtr); } uptr Size = Chunk->getUsableSize(&OldHeader); if (OldHeader.AllocType != FromMalloc) { dieWithMessage("ERROR: invalid chunk type when reallocating address %p\n", Chunk); } UnpackedHeader NewHeader = OldHeader; // The new size still fits in the current chunk. if (NewSize <= Size) { NewHeader.UnusedBytes = Size - NewSize; Chunk->compareExchangeHeader(&NewHeader, &OldHeader); return OldPtr; } // Otherwise, we have to allocate a new chunk and copy the contents of the // old one. void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc); if (NewPtr) { uptr OldSize = Size - OldHeader.UnusedBytes; memcpy(NewPtr, OldPtr, Min(NewSize, OldSize)); NewHeader.State = ChunkQuarantine; Chunk->compareExchangeHeader(&NewHeader, &OldHeader); if (LIKELY(!ThreadTornDown)) { AllocatorQuarantine.Put(&ThreadQuarantineCache, QuarantineCallback(&Cache), Chunk, Size); } else { SpinMutexLock l(&FallbackMutex); AllocatorQuarantine.Put(&FallbackQuarantineCache, QuarantineCallback(&FallbackAllocatorCache), Chunk, Size); } } return NewPtr; } // Helper function that returns the actual usable size of a chunk. uptr getUsableSize(const void *Ptr) { if (UNLIKELY(!ThreadInited)) initThread(); if (!Ptr) return 0; uptr ChunkBeg = reinterpret_cast(Ptr); ScudoChunk *Chunk = reinterpret_cast(ChunkBeg - AlignedChunkHeaderSize); UnpackedHeader Header; Chunk->loadHeader(&Header); // Getting the usable size of a chunk only makes sense if it's allocated. if (Header.State != ChunkAllocated) { dieWithMessage("ERROR: invalid chunk state when sizing address %p\n", Ptr); } return Chunk->getUsableSize(&Header); } void *calloc(uptr NMemB, uptr Size) { if (UNLIKELY(!ThreadInited)) initThread(); uptr Total = NMemB * Size; if (Size != 0 && Total / Size != NMemB) // Overflow check return BackendAllocator.ReturnNullOrDieOnBadRequest(); void *Ptr = allocate(Total, MinAlignment, FromMalloc); // If ZeroContents, the content of the chunk has already been zero'd out. if (!ZeroContents && Ptr && BackendAllocator.FromPrimary(Ptr)) memset(Ptr, 0, getUsableSize(Ptr)); return Ptr; } void drainQuarantine() { AllocatorQuarantine.Drain(&ThreadQuarantineCache, QuarantineCallback(&Cache)); } }; static Allocator Instance(LINKER_INITIALIZED); static ScudoAllocator &getAllocator() { return Instance.BackendAllocator; } void initAllocator(const AllocatorOptions &Options) { Instance.init(Options); } void drainQuarantine() { Instance.drainQuarantine(); } void *scudoMalloc(uptr Size, AllocType Type) { return Instance.allocate(Size, MinAlignment, Type); } void scudoFree(void *Ptr, AllocType Type) { Instance.deallocate(Ptr, 0, Type); } void scudoSizedFree(void *Ptr, uptr Size, AllocType Type) { Instance.deallocate(Ptr, Size, Type); } void *scudoRealloc(void *Ptr, uptr Size) { if (!Ptr) return Instance.allocate(Size, MinAlignment, FromMalloc); if (Size == 0) { Instance.deallocate(Ptr, 0, FromMalloc); return nullptr; } return Instance.reallocate(Ptr, Size); } void *scudoCalloc(uptr NMemB, uptr Size) { return Instance.calloc(NMemB, Size); } void *scudoValloc(uptr Size) { return Instance.allocate(Size, GetPageSizeCached(), FromMemalign); } void *scudoMemalign(uptr Alignment, uptr Size) { return Instance.allocate(Size, Alignment, FromMemalign); } void *scudoPvalloc(uptr Size) { uptr PageSize = GetPageSizeCached(); Size = RoundUpTo(Size, PageSize); if (Size == 0) { // pvalloc(0) should allocate one page. Size = PageSize; } return Instance.allocate(Size, PageSize, FromMemalign); } int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size) { *MemPtr = Instance.allocate(Size, Alignment, FromMemalign); return 0; } void *scudoAlignedAlloc(uptr Alignment, uptr Size) { // size must be a multiple of the alignment. To avoid a division, we first // make sure that alignment is a power of 2. CHECK(IsPowerOfTwo(Alignment)); CHECK_EQ((Size & (Alignment - 1)), 0); return Instance.allocate(Size, Alignment, FromMalloc); } uptr scudoMallocUsableSize(void *Ptr) { return Instance.getUsableSize(Ptr); } } // namespace __scudo using namespace __scudo; // MallocExtension helper functions uptr __sanitizer_get_current_allocated_bytes() { uptr stats[AllocatorStatCount]; getAllocator().GetStats(stats); return stats[AllocatorStatAllocated]; } uptr __sanitizer_get_heap_size() { uptr stats[AllocatorStatCount]; getAllocator().GetStats(stats); return stats[AllocatorStatMapped]; } uptr __sanitizer_get_free_bytes() { return 1; } uptr __sanitizer_get_unmapped_bytes() { return 1; } uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } int __sanitizer_get_ownership(const void *Ptr) { return Instance.isValidPointer(Ptr); } uptr __sanitizer_get_allocated_size(const void *Ptr) { return Instance.getUsableSize(Ptr); } Index: projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_crc32.cpp =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_crc32.cpp (nonexistent) +++ projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_crc32.cpp (revision 312198) @@ -0,0 +1,53 @@ +//===-- scudo_crc32.cpp -----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// CRC32 function leveraging hardware specific instructions. This has to be +/// kept separated to restrict the use of compiler specific flags to this file. +/// +//===----------------------------------------------------------------------===// + +// Hardware CRC32 is supported at compilation via the following: +// - for i386 & x86_64: -msse4.2 +// - for ARM & AArch64: -march=armv8-a+crc or -mcrc +// An additional check must be performed at runtime as well to make sure the +// emitted instructions are valid on the target host. +#include "scudo_crc32.h" +#include "scudo_utils.h" + +#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) +# ifdef __SSE4_2__ +# include +# define CRC32_INTRINSIC FIRST_32_SECOND_64(_mm_crc32_u32, _mm_crc32_u64) +# endif +# ifdef __ARM_FEATURE_CRC32 +# include +# define CRC32_INTRINSIC FIRST_32_SECOND_64(__crc32cw, __crc32cd) +# endif +#endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) + +namespace __scudo { + +#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) +INLINE u32 computeHardwareCRC32(u32 Crc, uptr Data) { + return CRC32_INTRINSIC(Crc, Data); +} + +u32 computeCRC32(u32 Crc, uptr Data, u8 HashType) { + if (HashType == CRC32Hardware) { + return computeHardwareCRC32(Crc, Data); + } + return computeSoftwareCRC32(Crc, Data); +} +#else +u32 computeCRC32(u32 Crc, uptr Data, u8 HashType) { + return computeSoftwareCRC32(Crc, Data); +} +#endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) + +} // namespace __scudo Property changes on: projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_crc32.cpp ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_crc32.h =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_crc32.h (nonexistent) +++ projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_crc32.h (revision 312198) @@ -0,0 +1,30 @@ +//===-- scudo_crc32.h -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Header for scudo_crc32.cpp. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_CRC32_H_ +#define SCUDO_CRC32_H_ + +#include "sanitizer_common/sanitizer_internal_defs.h" + +namespace __scudo { + +enum : u8 { + CRC32Software = 0, + CRC32Hardware = 1, +}; + +u32 computeCRC32(u32 Crc, uptr Data, u8 HashType); + +} // namespace __scudo + +#endif // SCUDO_CRC32_H_ Property changes on: projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_crc32.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_utils.cpp =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_utils.cpp (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_utils.cpp (revision 312198) @@ -1,197 +1,196 @@ //===-- scudo_utils.cpp -----------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// /// /// Platform specific utility functions. /// //===----------------------------------------------------------------------===// #include "scudo_utils.h" #include #include #include #include #if defined(__x86_64__) || defined(__i386__) # include #endif #include // TODO(kostyak): remove __sanitizer *Printf uses in favor for our own less // complicated string formatting code. The following is a // temporary workaround to be able to use __sanitizer::VSNPrintf. namespace __sanitizer { extern int VSNPrintf(char *buff, int buff_length, const char *format, va_list args); } // namespace __sanitizer namespace __scudo { FORMAT(1, 2) void NORETURN dieWithMessage(const char *Format, ...) { // Our messages are tiny, 256 characters is more than enough. char Message[256]; va_list Args; va_start(Args, Format); __sanitizer::VSNPrintf(Message, sizeof(Message), Format, Args); va_end(Args); RawWrite(Message); Die(); } #if defined(__x86_64__) || defined(__i386__) // i386 and x86_64 specific code to detect CRC32 hardware support via CPUID. // CRC32 requires the SSE 4.2 instruction set. typedef struct { u32 Eax; u32 Ebx; u32 Ecx; u32 Edx; } CPUIDRegs; static void getCPUID(CPUIDRegs *Regs, u32 Level) { __get_cpuid(Level, &Regs->Eax, &Regs->Ebx, &Regs->Ecx, &Regs->Edx); } CPUIDRegs getCPUFeatures() { CPUIDRegs VendorRegs = {}; getCPUID(&VendorRegs, 0); bool IsIntel = (VendorRegs.Ebx == signature_INTEL_ebx) && (VendorRegs.Edx == signature_INTEL_edx) && (VendorRegs.Ecx == signature_INTEL_ecx); bool IsAMD = (VendorRegs.Ebx == signature_AMD_ebx) && (VendorRegs.Edx == signature_AMD_edx) && (VendorRegs.Ecx == signature_AMD_ecx); // Default to an empty feature set if not on a supported CPU. CPUIDRegs FeaturesRegs = {}; if (IsIntel || IsAMD) { getCPUID(&FeaturesRegs, 1); } return FeaturesRegs; } #ifndef bit_SSE4_2 #define bit_SSE4_2 bit_SSE42 // clang and gcc have different defines. #endif bool testCPUFeature(CPUFeature Feature) { static CPUIDRegs FeaturesRegs = getCPUFeatures(); switch (Feature) { case CRC32CPUFeature: // CRC32 is provided by SSE 4.2. return !!(FeaturesRegs.Ecx & bit_SSE4_2); default: break; } return false; } #else bool testCPUFeature(CPUFeature Feature) { return false; } #endif // defined(__x86_64__) || defined(__i386__) // readRetry will attempt to read Count bytes from the Fd specified, and if // interrupted will retry to read additional bytes to reach Count. static ssize_t readRetry(int Fd, u8 *Buffer, size_t Count) { ssize_t AmountRead = 0; while (static_cast(AmountRead) < Count) { ssize_t Result = read(Fd, Buffer + AmountRead, Count - AmountRead); if (Result > 0) AmountRead += Result; else if (!Result) break; else if (errno != EINTR) { AmountRead = -1; break; } } return AmountRead; } static void fillRandom(u8 *Data, ssize_t Size) { int Fd = open("/dev/urandom", O_RDONLY); if (Fd < 0) { dieWithMessage("ERROR: failed to open /dev/urandom.\n"); } bool Success = readRetry(Fd, Data, Size) == Size; close(Fd); if (!Success) { dieWithMessage("ERROR: failed to read enough data from /dev/urandom.\n"); } } // Default constructor for Xorshift128Plus seeds the state with /dev/urandom. // TODO(kostyak): investigate using getrandom() if available. Xorshift128Plus::Xorshift128Plus() { fillRandom(reinterpret_cast(State), sizeof(State)); } const static u32 CRC32Table[] = { 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d }; -u32 computeCRC32(u32 Crc, uptr Data) -{ +u32 computeSoftwareCRC32(u32 Crc, uptr Data) { for (uptr i = 0; i < sizeof(Data); i++) { Crc = CRC32Table[(Crc ^ Data) & 0xff] ^ (Crc >> 8); Data >>= 8; } return Crc; } } // namespace __scudo Index: projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_utils.h =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_utils.h (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/scudo/scudo_utils.h (revision 312198) @@ -1,61 +1,61 @@ //===-- scudo_utils.h -------------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// /// /// Header for scudo_utils.cpp. /// //===----------------------------------------------------------------------===// #ifndef SCUDO_UTILS_H_ #define SCUDO_UTILS_H_ #include #include "sanitizer_common/sanitizer_common.h" namespace __scudo { template inline Dest bit_cast(const Source& source) { static_assert(sizeof(Dest) == sizeof(Source), "Sizes are not equal!"); Dest dest; memcpy(&dest, &source, sizeof(dest)); return dest; } void NORETURN dieWithMessage(const char *Format, ...); enum CPUFeature { CRC32CPUFeature = 0, MaxCPUFeature, }; bool testCPUFeature(CPUFeature feature); // Tiny PRNG based on https://en.wikipedia.org/wiki/Xorshift#xorshift.2B // The state (128 bits) will be stored in thread local storage. struct Xorshift128Plus { public: Xorshift128Plus(); u64 Next() { u64 x = State[0]; const u64 y = State[1]; State[0] = y; x ^= x << 23; State[1] = x ^ y ^ (x >> 17) ^ (y >> 26); return State[1] + y; } private: u64 State[2]; }; -// Software CRC32 functions, to be used when SSE 4.2 support is not detected. -u32 computeCRC32(u32 Crc, uptr Data); +// Software CRC32 functions, to be used when hardware support is not detected. +u32 computeSoftwareCRC32(u32 Crc, uptr Data); } // namespace __scudo #endif // SCUDO_UTILS_H_ Index: projects/clang400-import/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.inc =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.inc (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/tsan/rtl/tsan_flags.inc (revision 312198) @@ -1,83 +1,86 @@ //===-- tsan_flags.inc ------------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // TSan runtime flags. // //===----------------------------------------------------------------------===// #ifndef TSAN_FLAG # error "Define TSAN_FLAG prior to including this file!" #endif // TSAN_FLAG(Type, Name, DefaultValue, Description) // See COMMON_FLAG in sanitizer_flags.inc for more details. TSAN_FLAG(bool, enable_annotations, true, "Enable dynamic annotations, otherwise they are no-ops.") // Suppress a race report if we've already output another race report // with the same stack. TSAN_FLAG(bool, suppress_equal_stacks, true, "Suppress a race report if we've already output another race report " "with the same stack.") TSAN_FLAG(bool, suppress_equal_addresses, true, "Suppress a race report if we've already output another race report " "on the same address.") TSAN_FLAG(bool, report_bugs, true, "Turns off bug reporting entirely (useful for benchmarking).") TSAN_FLAG(bool, report_thread_leaks, true, "Report thread leaks at exit?") TSAN_FLAG(bool, report_destroy_locked, true, "Report destruction of a locked mutex?") TSAN_FLAG(bool, report_mutex_bugs, true, "Report incorrect usages of mutexes and mutex annotations?") TSAN_FLAG(bool, report_signal_unsafe, true, "Report violations of async signal-safety " "(e.g. malloc() call from a signal handler).") TSAN_FLAG(bool, report_atomic_races, true, "Report races between atomic and plain memory accesses.") TSAN_FLAG( bool, force_seq_cst_atomics, false, "If set, all atomics are effectively sequentially consistent (seq_cst), " "regardless of what user actually specified.") TSAN_FLAG(bool, print_benign, false, "Print matched \"benign\" races at exit.") TSAN_FLAG(bool, halt_on_error, false, "Exit after first reported error.") TSAN_FLAG(int, atexit_sleep_ms, 1000, "Sleep in main thread before exiting for that many ms " "(useful to catch \"at exit\" races).") TSAN_FLAG(const char *, profile_memory, "", "If set, periodically write memory profile to that file.") TSAN_FLAG(int, flush_memory_ms, 0, "Flush shadow memory every X ms.") TSAN_FLAG(int, flush_symbolizer_ms, 5000, "Flush symbolizer caches every X ms.") TSAN_FLAG( int, memory_limit_mb, 0, "Resident memory limit in MB to aim at." "If the process consumes more memory, then TSan will flush shadow memory.") TSAN_FLAG(bool, stop_on_start, false, "Stops on start until __tsan_resume() is called (for debugging).") TSAN_FLAG(bool, running_on_valgrind, false, "Controls whether RunningOnValgrind() returns true or false.") // There are a lot of goroutines in Go, so we use smaller history. TSAN_FLAG( int, history_size, SANITIZER_GO ? 1 : 3, "Per-thread history size, controls how many previous memory accesses " "are remembered per thread. Possible values are [0..7]. " "history_size=0 amounts to 32K memory accesses. Each next value doubles " "the amount of memory accesses, up to history_size=7 that amounts to " "4M memory accesses. The default value is 2 (128K memory accesses).") TSAN_FLAG(int, io_sync, 1, "Controls level of synchronization implied by IO operations. " "0 - no synchronization " "1 - reasonable level of synchronization (write->read)" "2 - global synchronization of all IO operations.") TSAN_FLAG(bool, die_after_fork, true, "Die after multi-threaded fork if the child creates new threads.") TSAN_FLAG(const char *, suppressions, "", "Suppressions file name.") TSAN_FLAG(bool, ignore_interceptors_accesses, false, "Ignore reads and writes from all interceptors.") +TSAN_FLAG(bool, ignore_noninstrumented_modules, false, + "Interceptors should only detect races when called from instrumented " + "modules.") TSAN_FLAG(bool, shared_ptr_interceptor, true, "Track atomic reference counting in libc++ shared_ptr and weak_ptr.") Index: projects/clang400-import/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc (revision 312198) @@ -1,2660 +1,2655 @@ //===-- tsan_interceptors.cc ----------------------------------------------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file is a part of ThreadSanitizer (TSan), a race detector. // // FIXME: move as many interceptors as possible into // sanitizer_common/sanitizer_common_interceptors.inc //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_stacktrace.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" #include "interception/interception.h" #include "tsan_interceptors.h" #include "tsan_interface.h" #include "tsan_platform.h" #include "tsan_suppressions.h" #include "tsan_rtl.h" #include "tsan_mman.h" #include "tsan_fd.h" #if SANITIZER_POSIX #include "sanitizer_common/sanitizer_posix.h" #endif using namespace __tsan; // NOLINT #if SANITIZER_FREEBSD || SANITIZER_MAC #define __errno_location __error #define stdout __stdoutp #define stderr __stderrp #endif #if SANITIZER_ANDROID #define __errno_location __errno #define mallopt(a, b) #endif #if SANITIZER_LINUX || SANITIZER_FREEBSD #define PTHREAD_CREATE_DETACHED 1 #elif SANITIZER_MAC #define PTHREAD_CREATE_DETACHED 2 #endif #ifdef __mips__ const int kSigCount = 129; #else const int kSigCount = 65; #endif struct my_siginfo_t { // The size is determined by looking at sizeof of real siginfo_t on linux. u64 opaque[128 / sizeof(u64)]; }; #ifdef __mips__ struct ucontext_t { u64 opaque[768 / sizeof(u64) + 1]; }; #else struct ucontext_t { // The size is determined by looking at sizeof of real ucontext_t on linux. u64 opaque[936 / sizeof(u64) + 1]; }; #endif #if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 #define PTHREAD_ABI_BASE "GLIBC_2.3.2" #elif defined(__aarch64__) || SANITIZER_PPC64V2 #define PTHREAD_ABI_BASE "GLIBC_2.17" #endif extern "C" int pthread_attr_init(void *attr); extern "C" int pthread_attr_destroy(void *attr); DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *) extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize); extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v)); extern "C" int pthread_setspecific(unsigned key, const void *v); DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *) DECLARE_REAL(int, fflush, __sanitizer_FILE *fp) DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size) DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) extern "C" void *pthread_self(); extern "C" void _exit(int status); extern "C" int *__errno_location(); extern "C" int fileno_unlocked(void *stream); extern "C" int dirfd(void *dirp); #if !SANITIZER_FREEBSD && !SANITIZER_ANDROID extern "C" int mallopt(int param, int value); #endif extern __sanitizer_FILE *stdout, *stderr; #if !SANITIZER_FREEBSD && !SANITIZER_MAC const int PTHREAD_MUTEX_RECURSIVE = 1; const int PTHREAD_MUTEX_RECURSIVE_NP = 1; #else const int PTHREAD_MUTEX_RECURSIVE = 2; const int PTHREAD_MUTEX_RECURSIVE_NP = 2; #endif const int EINVAL = 22; const int EBUSY = 16; const int EOWNERDEAD = 130; #if !SANITIZER_FREEBSD && !SANITIZER_MAC const int EPOLL_CTL_ADD = 1; #endif const int SIGILL = 4; const int SIGABRT = 6; const int SIGFPE = 8; const int SIGSEGV = 11; const int SIGPIPE = 13; const int SIGTERM = 15; #if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC const int SIGBUS = 10; const int SIGSYS = 12; #else const int SIGBUS = 7; const int SIGSYS = 31; #endif void *const MAP_FAILED = (void*)-1; #if !SANITIZER_MAC const int PTHREAD_BARRIER_SERIAL_THREAD = -1; #endif const int MAP_FIXED = 0x10; typedef long long_t; // NOLINT // From /usr/include/unistd.h # define F_ULOCK 0 /* Unlock a previously locked region. */ # define F_LOCK 1 /* Lock a region for exclusive use. */ # define F_TLOCK 2 /* Test and lock a region for exclusive use. */ # define F_TEST 3 /* Test a region for other processes locks. */ #define errno (*__errno_location()) typedef void (*sighandler_t)(int sig); typedef void (*sigactionhandler_t)(int sig, my_siginfo_t *siginfo, void *uctx); #if SANITIZER_ANDROID struct sigaction_t { u32 sa_flags; union { sighandler_t sa_handler; sigactionhandler_t sa_sigaction; }; __sanitizer_sigset_t sa_mask; void (*sa_restorer)(); }; #else struct sigaction_t { #ifdef __mips__ u32 sa_flags; #endif union { sighandler_t sa_handler; sigactionhandler_t sa_sigaction; }; #if SANITIZER_FREEBSD int sa_flags; __sanitizer_sigset_t sa_mask; #elif SANITIZER_MAC __sanitizer_sigset_t sa_mask; int sa_flags; #else __sanitizer_sigset_t sa_mask; #ifndef __mips__ int sa_flags; #endif void (*sa_restorer)(); #endif }; #endif const sighandler_t SIG_DFL = (sighandler_t)0; const sighandler_t SIG_IGN = (sighandler_t)1; const sighandler_t SIG_ERR = (sighandler_t)-1; #if SANITIZER_FREEBSD || SANITIZER_MAC const int SA_SIGINFO = 0x40; const int SIG_SETMASK = 3; #elif defined(__mips__) const int SA_SIGINFO = 8; const int SIG_SETMASK = 3; #else const int SA_SIGINFO = 4; const int SIG_SETMASK = 2; #endif #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \ (!cur_thread()->is_inited) static sigaction_t sigactions[kSigCount]; namespace __tsan { struct SignalDesc { bool armed; bool sigaction; my_siginfo_t siginfo; ucontext_t ctx; }; struct ThreadSignalContext { int int_signal_send; atomic_uintptr_t in_blocking_func; atomic_uintptr_t have_pending_signals; SignalDesc pending_signals[kSigCount]; // emptyset and oldset are too big for stack. __sanitizer_sigset_t emptyset; __sanitizer_sigset_t oldset; }; // The object is 64-byte aligned, because we want hot data to be located in // a single cache line if possible (it's accessed in every interceptor). static ALIGNED(64) char libignore_placeholder[sizeof(LibIgnore)]; static LibIgnore *libignore() { return reinterpret_cast(&libignore_placeholder[0]); } void InitializeLibIgnore() { const SuppressionContext &supp = *Suppressions(); const uptr n = supp.SuppressionCount(); for (uptr i = 0; i < n; i++) { const Suppression *s = supp.SuppressionAt(i); if (0 == internal_strcmp(s->type, kSuppressionLib)) libignore()->AddIgnoredLibrary(s->templ); } + if (flags()->ignore_noninstrumented_modules) + libignore()->IgnoreNoninstrumentedModules(true); libignore()->OnLibraryLoaded(0); } } // namespace __tsan static ThreadSignalContext *SigCtx(ThreadState *thr) { ThreadSignalContext *ctx = (ThreadSignalContext*)thr->signal_ctx; if (ctx == 0 && !thr->is_dead) { ctx = (ThreadSignalContext*)MmapOrDie(sizeof(*ctx), "ThreadSignalContext"); MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx)); thr->signal_ctx = ctx; } return ctx; } #if !SANITIZER_MAC static unsigned g_thread_finalize_key; #endif ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc) - : thr_(thr) - , pc_(pc) - , in_ignored_lib_(false) { + : thr_(thr), pc_(pc), in_ignored_lib_(false), ignoring_(false) { Initialize(thr); - if (!thr_->is_inited) - return; - if (!thr_->ignore_interceptors) - FuncEntry(thr, pc); + if (!thr_->is_inited) return; + if (!thr_->ignore_interceptors) FuncEntry(thr, pc); DPrintf("#%d: intercept %s()\n", thr_->tid, fname); - if (!thr_->in_ignored_lib && libignore()->IsIgnored(pc)) { - in_ignored_lib_ = true; - thr_->in_ignored_lib = true; - ThreadIgnoreBegin(thr_, pc_); - } - if (flags()->ignore_interceptors_accesses) ThreadIgnoreBegin(thr_, pc_); + ignoring_ = + !thr_->in_ignored_lib && (flags()->ignore_interceptors_accesses || + libignore()->IsIgnored(pc, &in_ignored_lib_)); + EnableIgnores(); } ScopedInterceptor::~ScopedInterceptor() { - if (!thr_->is_inited) - return; - if (flags()->ignore_interceptors_accesses) ThreadIgnoreEnd(thr_, pc_); - if (in_ignored_lib_) { - thr_->in_ignored_lib = false; - ThreadIgnoreEnd(thr_, pc_); - } + if (!thr_->is_inited) return; + DisableIgnores(); if (!thr_->ignore_interceptors) { ProcessPendingSignals(thr_); FuncExit(thr_); CheckNoLocks(thr_); } } -void ScopedInterceptor::UserCallbackStart() { - if (flags()->ignore_interceptors_accesses) ThreadIgnoreEnd(thr_, pc_); - if (in_ignored_lib_) { - thr_->in_ignored_lib = false; - ThreadIgnoreEnd(thr_, pc_); +void ScopedInterceptor::EnableIgnores() { + if (ignoring_) { + ThreadIgnoreBegin(thr_, pc_); + if (in_ignored_lib_) { + DCHECK(!thr_->in_ignored_lib); + thr_->in_ignored_lib = true; + } } } -void ScopedInterceptor::UserCallbackEnd() { - if (in_ignored_lib_) { - thr_->in_ignored_lib = true; - ThreadIgnoreBegin(thr_, pc_); +void ScopedInterceptor::DisableIgnores() { + if (ignoring_) { + ThreadIgnoreEnd(thr_, pc_); + if (in_ignored_lib_) { + DCHECK(thr_->in_ignored_lib); + thr_->in_ignored_lib = false; + } } - if (flags()->ignore_interceptors_accesses) ThreadIgnoreBegin(thr_, pc_); } #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) #if SANITIZER_FREEBSD # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) #else # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver) #endif #define READ_STRING_OF_LEN(thr, pc, s, len, n) \ MemoryAccessRange((thr), (pc), (uptr)(s), \ common_flags()->strict_string_checks ? (len) + 1 : (n), false) #define READ_STRING(thr, pc, s, n) \ READ_STRING_OF_LEN((thr), (pc), (s), internal_strlen(s), (n)) #define BLOCK_REAL(name) (BlockingCall(thr), REAL(name)) struct BlockingCall { explicit BlockingCall(ThreadState *thr) : thr(thr) , ctx(SigCtx(thr)) { for (;;) { atomic_store(&ctx->in_blocking_func, 1, memory_order_relaxed); if (atomic_load(&ctx->have_pending_signals, memory_order_relaxed) == 0) break; atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); ProcessPendingSignals(thr); } // When we are in a "blocking call", we process signals asynchronously // (right when they arrive). In this context we do not expect to be // executing any user/runtime code. The known interceptor sequence when // this is not true is: pthread_join -> munmap(stack). It's fine // to ignore munmap in this case -- we handle stack shadow separately. thr->ignore_interceptors++; } ~BlockingCall() { thr->ignore_interceptors--; atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); } ThreadState *thr; ThreadSignalContext *ctx; }; TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) { SCOPED_TSAN_INTERCEPTOR(sleep, sec); unsigned res = BLOCK_REAL(sleep)(sec); AfterSleep(thr, pc); return res; } TSAN_INTERCEPTOR(int, usleep, long_t usec) { SCOPED_TSAN_INTERCEPTOR(usleep, usec); int res = BLOCK_REAL(usleep)(usec); AfterSleep(thr, pc); return res; } TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) { SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem); int res = BLOCK_REAL(nanosleep)(req, rem); AfterSleep(thr, pc); return res; } // The sole reason tsan wraps atexit callbacks is to establish synchronization // between callback setup and callback execution. struct AtExitCtx { void (*f)(); void *arg; }; static void at_exit_wrapper(void *arg) { ThreadState *thr = cur_thread(); uptr pc = 0; Acquire(thr, pc, (uptr)arg); AtExitCtx *ctx = (AtExitCtx*)arg; ((void(*)(void *arg))ctx->f)(ctx->arg); InternalFree(ctx); } static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), void *arg, void *dso); #if !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, atexit, void (*f)()) { if (cur_thread()->in_symbolizer) return 0; // We want to setup the atexit callback even if we are in ignored lib // or after fork. SCOPED_INTERCEPTOR_RAW(atexit, f); return setup_at_exit_wrapper(thr, pc, (void(*)())f, 0, 0); } #endif TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { if (cur_thread()->in_symbolizer) return 0; SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso); } static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), void *arg, void *dso) { AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); ctx->f = f; ctx->arg = arg; Release(thr, pc, (uptr)ctx); // Memory allocation in __cxa_atexit will race with free during exit, // because we do not see synchronization around atexit callback list. ThreadIgnoreBegin(thr, pc); int res = REAL(__cxa_atexit)(at_exit_wrapper, ctx, dso); ThreadIgnoreEnd(thr, pc); return res; } #if !SANITIZER_MAC static void on_exit_wrapper(int status, void *arg) { ThreadState *thr = cur_thread(); uptr pc = 0; Acquire(thr, pc, (uptr)arg); AtExitCtx *ctx = (AtExitCtx*)arg; ((void(*)(int status, void *arg))ctx->f)(status, ctx->arg); InternalFree(ctx); } TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { if (cur_thread()->in_symbolizer) return 0; SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); ctx->f = (void(*)())f; ctx->arg = arg; Release(thr, pc, (uptr)ctx); // Memory allocation in __cxa_atexit will race with free during exit, // because we do not see synchronization around atexit callback list. ThreadIgnoreBegin(thr, pc); int res = REAL(on_exit)(on_exit_wrapper, ctx); ThreadIgnoreEnd(thr, pc); return res; } #endif // Cleanup old bufs. static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) { for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { JmpBuf *buf = &thr->jmp_bufs[i]; if (buf->sp <= sp) { uptr sz = thr->jmp_bufs.Size(); internal_memcpy(buf, &thr->jmp_bufs[sz - 1], sizeof(*buf)); thr->jmp_bufs.PopBack(); i--; } } } static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) { if (!thr->is_inited) // called from libc guts during bootstrap return; // Cleanup old bufs. JmpBufGarbageCollect(thr, sp); // Remember the buf. JmpBuf *buf = thr->jmp_bufs.PushBack(); buf->sp = sp; buf->mangled_sp = mangled_sp; buf->shadow_stack_pos = thr->shadow_stack_pos; ThreadSignalContext *sctx = SigCtx(thr); buf->int_signal_send = sctx ? sctx->int_signal_send : 0; buf->in_blocking_func = sctx ? atomic_load(&sctx->in_blocking_func, memory_order_relaxed) : false; buf->in_signal_handler = atomic_load(&thr->in_signal_handler, memory_order_relaxed); } static void LongJmp(ThreadState *thr, uptr *env) { #ifdef __powerpc__ uptr mangled_sp = env[0]; #elif SANITIZER_FREEBSD || SANITIZER_MAC uptr mangled_sp = env[2]; #elif defined(SANITIZER_LINUX) # ifdef __aarch64__ uptr mangled_sp = env[13]; # elif defined(__mips64) uptr mangled_sp = env[1]; # else uptr mangled_sp = env[6]; # endif #endif // Find the saved buf by mangled_sp. for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { JmpBuf *buf = &thr->jmp_bufs[i]; if (buf->mangled_sp == mangled_sp) { CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos); // Unwind the stack. while (thr->shadow_stack_pos > buf->shadow_stack_pos) FuncExit(thr); ThreadSignalContext *sctx = SigCtx(thr); if (sctx) { sctx->int_signal_send = buf->int_signal_send; atomic_store(&sctx->in_blocking_func, buf->in_blocking_func, memory_order_relaxed); } atomic_store(&thr->in_signal_handler, buf->in_signal_handler, memory_order_relaxed); JmpBufGarbageCollect(thr, buf->sp - 1); // do not collect buf->sp return; } } Printf("ThreadSanitizer: can't find longjmp buf\n"); CHECK(0); } // FIXME: put everything below into a common extern "C" block? extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) { SetJmp(cur_thread(), sp, mangled_sp); } #if SANITIZER_MAC TSAN_INTERCEPTOR(int, setjmp, void *env); TSAN_INTERCEPTOR(int, _setjmp, void *env); TSAN_INTERCEPTOR(int, sigsetjmp, void *env); #else // SANITIZER_MAC // Not called. Merely to satisfy TSAN_INTERCEPT(). extern "C" SANITIZER_INTERFACE_ATTRIBUTE int __interceptor_setjmp(void *env); extern "C" int __interceptor_setjmp(void *env) { CHECK(0); return 0; } // FIXME: any reason to have a separate declaration? extern "C" SANITIZER_INTERFACE_ATTRIBUTE int __interceptor__setjmp(void *env); extern "C" int __interceptor__setjmp(void *env) { CHECK(0); return 0; } extern "C" SANITIZER_INTERFACE_ATTRIBUTE int __interceptor_sigsetjmp(void *env); extern "C" int __interceptor_sigsetjmp(void *env) { CHECK(0); return 0; } extern "C" SANITIZER_INTERFACE_ATTRIBUTE int __interceptor___sigsetjmp(void *env); extern "C" int __interceptor___sigsetjmp(void *env) { CHECK(0); return 0; } extern "C" int setjmp(void *env); extern "C" int _setjmp(void *env); extern "C" int sigsetjmp(void *env); extern "C" int __sigsetjmp(void *env); DEFINE_REAL(int, setjmp, void *env) DEFINE_REAL(int, _setjmp, void *env) DEFINE_REAL(int, sigsetjmp, void *env) DEFINE_REAL(int, __sigsetjmp, void *env) #endif // SANITIZER_MAC TSAN_INTERCEPTOR(void, longjmp, uptr *env, int val) { // Note: if we call REAL(longjmp) in the context of ScopedInterceptor, // bad things will happen. We will jump over ScopedInterceptor dtor and can // leave thr->in_ignored_lib set. { SCOPED_INTERCEPTOR_RAW(longjmp, env, val); } LongJmp(cur_thread(), env); REAL(longjmp)(env, val); } TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) { { SCOPED_INTERCEPTOR_RAW(siglongjmp, env, val); } LongJmp(cur_thread(), env); REAL(siglongjmp)(env, val); } #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, malloc, uptr size) { if (cur_thread()->in_symbolizer) return InternalAlloc(size); void *p = 0; { SCOPED_INTERCEPTOR_RAW(malloc, size); p = user_alloc(thr, pc, size); } invoke_malloc_hook(p, size); return p; } TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz); return user_alloc(thr, pc, sz, align); } TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { if (cur_thread()->in_symbolizer) return InternalCalloc(size, n); void *p = 0; { SCOPED_INTERCEPTOR_RAW(calloc, size, n); p = user_calloc(thr, pc, size, n); } invoke_malloc_hook(p, n * size); return p; } TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { if (cur_thread()->in_symbolizer) return InternalRealloc(p, size); if (p) invoke_free_hook(p); { SCOPED_INTERCEPTOR_RAW(realloc, p, size); p = user_realloc(thr, pc, p, size); } invoke_malloc_hook(p, size); return p; } TSAN_INTERCEPTOR(void, free, void *p) { if (p == 0) return; if (cur_thread()->in_symbolizer) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(free, p); user_free(thr, pc, p); } TSAN_INTERCEPTOR(void, cfree, void *p) { if (p == 0) return; if (cur_thread()->in_symbolizer) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(cfree, p); user_free(thr, pc, p); } TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) { SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p); return user_alloc_usable_size(p); } #endif TSAN_INTERCEPTOR(char*, strcpy, char *dst, const char *src) { // NOLINT SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src); // NOLINT uptr srclen = internal_strlen(src); MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true); MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false); return REAL(strcpy)(dst, src); // NOLINT } TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) { SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n); uptr srclen = internal_strnlen(src, n); MemoryAccessRange(thr, pc, (uptr)dst, n, true); MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false); return REAL(strncpy)(dst, src, n); } TSAN_INTERCEPTOR(char*, strdup, const char *str) { SCOPED_TSAN_INTERCEPTOR(strdup, str); // strdup will call malloc, so no instrumentation is required here. return REAL(strdup)(str); } static bool fix_mmap_addr(void **addr, long_t sz, int flags) { if (*addr) { if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) { if (flags & MAP_FIXED) { errno = EINVAL; return false; } else { *addr = 0; } } } return true; } TSAN_INTERCEPTOR(void *, mmap, void *addr, SIZE_T sz, int prot, int flags, int fd, OFF_T off) { SCOPED_TSAN_INTERCEPTOR(mmap, addr, sz, prot, flags, fd, off); if (!fix_mmap_addr(&addr, sz, flags)) return MAP_FAILED; void *res = REAL(mmap)(addr, sz, prot, flags, fd, off); if (res != MAP_FAILED) { if (fd > 0) FdAccess(thr, pc, fd); if (thr->ignore_reads_and_writes == 0) MemoryRangeImitateWrite(thr, pc, (uptr)res, sz); else MemoryResetRange(thr, pc, (uptr)res, sz); } return res; } #if SANITIZER_LINUX TSAN_INTERCEPTOR(void *, mmap64, void *addr, SIZE_T sz, int prot, int flags, int fd, OFF64_T off) { SCOPED_TSAN_INTERCEPTOR(mmap64, addr, sz, prot, flags, fd, off); if (!fix_mmap_addr(&addr, sz, flags)) return MAP_FAILED; void *res = REAL(mmap64)(addr, sz, prot, flags, fd, off); if (res != MAP_FAILED) { if (fd > 0) FdAccess(thr, pc, fd); if (thr->ignore_reads_and_writes == 0) MemoryRangeImitateWrite(thr, pc, (uptr)res, sz); else MemoryResetRange(thr, pc, (uptr)res, sz); } return res; } #define TSAN_MAYBE_INTERCEPT_MMAP64 TSAN_INTERCEPT(mmap64) #else #define TSAN_MAYBE_INTERCEPT_MMAP64 #endif TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz); if (sz != 0) { // If sz == 0, munmap will return EINVAL and don't unmap any memory. DontNeedShadowFor((uptr)addr, sz); ScopedGlobalProcessor sgp; ctx->metamap.ResetRange(thr->proc(), (uptr)addr, (uptr)sz); } int res = REAL(munmap)(addr, sz); return res; } #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { SCOPED_INTERCEPTOR_RAW(memalign, align, sz); return user_alloc(thr, pc, sz, align); } #define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign) #else #define TSAN_MAYBE_INTERCEPT_MEMALIGN #endif #if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { SCOPED_INTERCEPTOR_RAW(memalign, align, sz); return user_alloc(thr, pc, sz, align); } TSAN_INTERCEPTOR(void*, valloc, uptr sz) { SCOPED_INTERCEPTOR_RAW(valloc, sz); return user_alloc(thr, pc, sz, GetPageSizeCached()); } #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { SCOPED_INTERCEPTOR_RAW(pvalloc, sz); sz = RoundUp(sz, GetPageSizeCached()); return user_alloc(thr, pc, sz, GetPageSizeCached()); } #define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc) #else #define TSAN_MAYBE_INTERCEPT_PVALLOC #endif #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz); *memptr = user_alloc(thr, pc, sz, align); return 0; } #endif // __cxa_guard_acquire and friends need to be intercepted in a special way - // regular interceptors will break statically-linked libstdc++. Linux // interceptors are especially defined as weak functions (so that they don't // cause link errors when user defines them as well). So they silently // auto-disable themselves when such symbol is already present in the binary. If // we link libstdc++ statically, it will bring own __cxa_guard_acquire which // will silently replace our interceptor. That's why on Linux we simply export // these interceptors with INTERFACE_ATTRIBUTE. // On OS X, we don't support statically linking, so we just use a regular // interceptor. #if SANITIZER_MAC #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR #else #define STDCXX_INTERCEPTOR(rettype, name, ...) \ extern "C" rettype INTERFACE_ATTRIBUTE name(__VA_ARGS__) #endif // Used in thread-safe function static initialization. STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) { SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g); for (;;) { u32 cmp = atomic_load(g, memory_order_acquire); if (cmp == 0) { if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed)) return 1; } else if (cmp == 1) { Acquire(thr, pc, (uptr)g); return 0; } else { internal_sched_yield(); } } } STDCXX_INTERCEPTOR(void, __cxa_guard_release, atomic_uint32_t *g) { SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g); Release(thr, pc, (uptr)g); atomic_store(g, 1, memory_order_release); } STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) { SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g); atomic_store(g, 0, memory_order_relaxed); } namespace __tsan { void DestroyThreadState() { ThreadState *thr = cur_thread(); Processor *proc = thr->proc(); ThreadFinish(thr); ProcUnwire(proc, thr); ProcDestroy(proc); ThreadSignalContext *sctx = thr->signal_ctx; if (sctx) { thr->signal_ctx = 0; UnmapOrDie(sctx, sizeof(*sctx)); } DTLS_Destroy(); cur_thread_finalize(); } } // namespace __tsan #if !SANITIZER_MAC static void thread_finalize(void *v) { uptr iter = (uptr)v; if (iter > 1) { if (pthread_setspecific(g_thread_finalize_key, (void*)(iter - 1))) { Printf("ThreadSanitizer: failed to set thread key\n"); Die(); } return; } DestroyThreadState(); } #endif struct ThreadParam { void* (*callback)(void *arg); void *param; atomic_uintptr_t tid; }; extern "C" void *__tsan_thread_start_func(void *arg) { ThreadParam *p = (ThreadParam*)arg; void* (*callback)(void *arg) = p->callback; void *param = p->param; int tid = 0; { ThreadState *thr = cur_thread(); // Thread-local state is not initialized yet. ScopedIgnoreInterceptors ignore; #if !SANITIZER_MAC ThreadIgnoreBegin(thr, 0); if (pthread_setspecific(g_thread_finalize_key, (void *)GetPthreadDestructorIterations())) { Printf("ThreadSanitizer: failed to set thread key\n"); Die(); } ThreadIgnoreEnd(thr, 0); #endif while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0) internal_sched_yield(); Processor *proc = ProcCreate(); ProcWire(proc, thr); ThreadStart(thr, tid, GetTid()); atomic_store(&p->tid, 0, memory_order_release); } void *res = callback(param); // Prevent the callback from being tail called, // it mixes up stack traces. volatile int foo = 42; foo++; return res; } TSAN_INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), void * param) { SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param); if (ctx->after_multithreaded_fork) { if (flags()->die_after_fork) { Report("ThreadSanitizer: starting new threads after multi-threaded " "fork is not supported. Dying (set die_after_fork=0 to override)\n"); Die(); } else { VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded " "fork is not supported (pid %d). Continuing because of " "die_after_fork=0, but you are on your own\n", internal_getpid()); } } __sanitizer_pthread_attr_t myattr; if (attr == 0) { pthread_attr_init(&myattr); attr = &myattr; } int detached = 0; REAL(pthread_attr_getdetachstate)(attr, &detached); AdjustStackSize(attr); ThreadParam p; p.callback = callback; p.param = param; atomic_store(&p.tid, 0, memory_order_relaxed); int res = -1; { // Otherwise we see false positives in pthread stack manipulation. ScopedIgnoreInterceptors ignore; ThreadIgnoreBegin(thr, pc); res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p); ThreadIgnoreEnd(thr, pc); } if (res == 0) { int tid = ThreadCreate(thr, pc, *(uptr*)th, detached == PTHREAD_CREATE_DETACHED); CHECK_NE(tid, 0); // Synchronization on p.tid serves two purposes: // 1. ThreadCreate must finish before the new thread starts. // Otherwise the new thread can call pthread_detach, but the pthread_t // identifier is not yet registered in ThreadRegistry by ThreadCreate. // 2. ThreadStart must finish before this thread continues. // Otherwise, this thread can call pthread_detach and reset thr->sync // before the new thread got a chance to acquire from it in ThreadStart. atomic_store(&p.tid, tid, memory_order_release); while (atomic_load(&p.tid, memory_order_acquire) != 0) internal_sched_yield(); } if (attr == &myattr) pthread_attr_destroy(&myattr); return res; } TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) { SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret); int tid = ThreadTid(thr, pc, (uptr)th); ThreadIgnoreBegin(thr, pc); int res = BLOCK_REAL(pthread_join)(th, ret); ThreadIgnoreEnd(thr, pc); if (res == 0) { ThreadJoin(thr, pc, tid); } return res; } DEFINE_REAL_PTHREAD_FUNCTIONS TSAN_INTERCEPTOR(int, pthread_detach, void *th) { SCOPED_TSAN_INTERCEPTOR(pthread_detach, th); int tid = ThreadTid(thr, pc, (uptr)th); int res = REAL(pthread_detach)(th); if (res == 0) { ThreadDetach(thr, pc, tid); } return res; } // Problem: // NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2). // pthread_cond_t has different size in the different versions. // If call new REAL functions for old pthread_cond_t, they will corrupt memory // after pthread_cond_t (old cond is smaller). // If we call old REAL functions for new pthread_cond_t, we will lose some // functionality (e.g. old functions do not support waiting against // CLOCK_REALTIME). // Proper handling would require to have 2 versions of interceptors as well. // But this is messy, in particular requires linker scripts when sanitizer // runtime is linked into a shared library. // Instead we assume we don't have dynamic libraries built against old // pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag // that allows to work with old libraries (but this mode does not support // some features, e.g. pthread_condattr_getpshared). static void *init_cond(void *c, bool force = false) { // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions. // So we allocate additional memory on the side large enough to hold // any pthread_cond_t object. Always call new REAL functions, but pass // the aux object to them. // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes // first word of pthread_cond_t to zero. // It's all relevant only for linux. if (!common_flags()->legacy_pthread_cond) return c; atomic_uintptr_t *p = (atomic_uintptr_t*)c; uptr cond = atomic_load(p, memory_order_acquire); if (!force && cond != 0) return (void*)cond; void *newcond = WRAP(malloc)(pthread_cond_t_sz); internal_memset(newcond, 0, pthread_cond_t_sz); if (atomic_compare_exchange_strong(p, &cond, (uptr)newcond, memory_order_acq_rel)) return newcond; WRAP(free)(newcond); return (void*)cond; } struct CondMutexUnlockCtx { ScopedInterceptor *si; ThreadState *thr; uptr pc; void *m; }; static void cond_mutex_unlock(CondMutexUnlockCtx *arg) { // pthread_cond_wait interceptor has enabled async signal delivery // (see BlockingCall below). Disable async signals since we are running // tsan code. Also ScopedInterceptor and BlockingCall destructors won't run // since the thread is cancelled, so we have to manually execute them // (the thread still can run some user code due to pthread_cleanup_push). ThreadSignalContext *ctx = SigCtx(arg->thr); CHECK_EQ(atomic_load(&ctx->in_blocking_func, memory_order_relaxed), 1); atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); MutexLock(arg->thr, arg->pc, (uptr)arg->m); // Undo BlockingCall ctor effects. arg->thr->ignore_interceptors--; arg->si->~ScopedInterceptor(); } INTERCEPTOR(int, pthread_cond_init, void *c, void *a) { void *cond = init_cond(c, true); SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, cond, a); MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); return REAL(pthread_cond_init)(cond, a); } static int cond_wait(ThreadState *thr, uptr pc, ScopedInterceptor *si, int (*fn)(void *c, void *m, void *abstime), void *c, void *m, void *t) { MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); MutexUnlock(thr, pc, (uptr)m); CondMutexUnlockCtx arg = {si, thr, pc, m}; int res = 0; // This ensures that we handle mutex lock even in case of pthread_cancel. // See test/tsan/cond_cancel.cc. { // Enable signal delivery while the thread is blocked. BlockingCall bc(thr); res = call_pthread_cancel_with_cleanup( fn, c, m, t, (void (*)(void *arg))cond_mutex_unlock, &arg); } if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); MutexLock(thr, pc, (uptr)m); return res; } INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m); return cond_wait(thr, pc, &si, (int (*)(void *c, void *m, void *abstime))REAL( pthread_cond_wait), cond, m, 0); } INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime); return cond_wait(thr, pc, &si, REAL(pthread_cond_timedwait), cond, m, abstime); } #if SANITIZER_MAC INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void *c, void *m, void *reltime) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait_relative_np, cond, m, reltime); return cond_wait(thr, pc, &si, REAL(pthread_cond_timedwait_relative_np), cond, m, reltime); } #endif INTERCEPTOR(int, pthread_cond_signal, void *c) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, cond); MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); return REAL(pthread_cond_signal)(cond); } INTERCEPTOR(int, pthread_cond_broadcast, void *c) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, cond); MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); return REAL(pthread_cond_broadcast)(cond); } INTERCEPTOR(int, pthread_cond_destroy, void *c) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, cond); MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); int res = REAL(pthread_cond_destroy)(cond); if (common_flags()->legacy_pthread_cond) { // Free our aux cond and zero the pointer to not leave dangling pointers. WRAP(free)(cond); atomic_store((atomic_uintptr_t*)c, 0, memory_order_relaxed); } return res; } TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a); int res = REAL(pthread_mutex_init)(m, a); if (res == 0) { bool recursive = false; if (a) { int type = 0; if (REAL(pthread_mutexattr_gettype)(a, &type) == 0) recursive = (type == PTHREAD_MUTEX_RECURSIVE || type == PTHREAD_MUTEX_RECURSIVE_NP); } MutexCreate(thr, pc, (uptr)m, false, recursive, false); } return res; } TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m); int res = REAL(pthread_mutex_destroy)(m); if (res == 0 || res == EBUSY) { MutexDestroy(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m); int res = REAL(pthread_mutex_trylock)(m); if (res == EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); if (res == 0 || res == EOWNERDEAD) MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true); return res; } #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime); int res = REAL(pthread_mutex_timedlock)(m, abstime); if (res == 0) { MutexLock(thr, pc, (uptr)m); } return res; } #endif #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared); int res = REAL(pthread_spin_init)(m, pshared); if (res == 0) { MutexCreate(thr, pc, (uptr)m, false, false, false); } return res; } TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m); int res = REAL(pthread_spin_destroy)(m); if (res == 0) { MutexDestroy(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m); int res = REAL(pthread_spin_lock)(m); if (res == 0) { MutexLock(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m); int res = REAL(pthread_spin_trylock)(m); if (res == 0) { MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true); } return res; } TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m); MutexUnlock(thr, pc, (uptr)m); int res = REAL(pthread_spin_unlock)(m); return res; } #endif TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a); int res = REAL(pthread_rwlock_init)(m, a); if (res == 0) { MutexCreate(thr, pc, (uptr)m, true, false, false); } return res; } TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m); int res = REAL(pthread_rwlock_destroy)(m); if (res == 0) { MutexDestroy(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m); int res = REAL(pthread_rwlock_rdlock)(m); if (res == 0) { MutexReadLock(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m); int res = REAL(pthread_rwlock_tryrdlock)(m); if (res == 0) { MutexReadLock(thr, pc, (uptr)m, /*try_lock=*/true); } return res; } #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime); int res = REAL(pthread_rwlock_timedrdlock)(m, abstime); if (res == 0) { MutexReadLock(thr, pc, (uptr)m); } return res; } #endif TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m); int res = REAL(pthread_rwlock_wrlock)(m); if (res == 0) { MutexLock(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m); int res = REAL(pthread_rwlock_trywrlock)(m); if (res == 0) { MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true); } return res; } #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime); int res = REAL(pthread_rwlock_timedwrlock)(m, abstime); if (res == 0) { MutexLock(thr, pc, (uptr)m); } return res; } #endif TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m); MutexReadOrWriteUnlock(thr, pc, (uptr)m); int res = REAL(pthread_rwlock_unlock)(m); return res; } #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) { SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count); MemoryWrite(thr, pc, (uptr)b, kSizeLog1); int res = REAL(pthread_barrier_init)(b, a, count); return res; } TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) { SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b); MemoryWrite(thr, pc, (uptr)b, kSizeLog1); int res = REAL(pthread_barrier_destroy)(b); return res; } TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) { SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b); Release(thr, pc, (uptr)b); MemoryRead(thr, pc, (uptr)b, kSizeLog1); int res = REAL(pthread_barrier_wait)(b); MemoryRead(thr, pc, (uptr)b, kSizeLog1); if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) { Acquire(thr, pc, (uptr)b); } return res; } #endif TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { SCOPED_INTERCEPTOR_RAW(pthread_once, o, f); if (o == 0 || f == 0) return EINVAL; atomic_uint32_t *a; if (!SANITIZER_MAC) a = static_cast(o); else // On OS X, pthread_once_t has a header with a long-sized signature. a = static_cast((void *)((char *)o + sizeof(long_t))); u32 v = atomic_load(a, memory_order_acquire); if (v == 0 && atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { (*f)(); if (!thr->in_ignored_lib) Release(thr, pc, (uptr)o); atomic_store(a, 2, memory_order_release); } else { while (v != 2) { internal_sched_yield(); v = atomic_load(a, memory_order_acquire); } if (!thr->in_ignored_lib) Acquire(thr, pc, (uptr)o); } return 0; } #if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf); if (fd > 0) FdAccess(thr, pc, fd); return REAL(__fxstat)(version, fd, buf); } #define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat) #else #define TSAN_MAYBE_INTERCEPT___FXSTAT #endif TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { #if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf); if (fd > 0) FdAccess(thr, pc, fd); return REAL(fstat)(fd, buf); #else SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf); if (fd > 0) FdAccess(thr, pc, fd); return REAL(__fxstat)(0, fd, buf); #endif } #if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) { SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf); if (fd > 0) FdAccess(thr, pc, fd); return REAL(__fxstat64)(version, fd, buf); } #define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64) #else #define TSAN_MAYBE_INTERCEPT___FXSTAT64 #endif #if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) { SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf); if (fd > 0) FdAccess(thr, pc, fd); return REAL(__fxstat64)(0, fd, buf); } #define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64) #else #define TSAN_MAYBE_INTERCEPT_FSTAT64 #endif TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) { SCOPED_TSAN_INTERCEPTOR(open, name, flags, mode); READ_STRING(thr, pc, name, 0); int fd = REAL(open)(name, flags, mode); if (fd >= 0) FdFileCreate(thr, pc, fd); return fd; } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) { SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode); READ_STRING(thr, pc, name, 0); int fd = REAL(open64)(name, flags, mode); if (fd >= 0) FdFileCreate(thr, pc, fd); return fd; } #define TSAN_MAYBE_INTERCEPT_OPEN64 TSAN_INTERCEPT(open64) #else #define TSAN_MAYBE_INTERCEPT_OPEN64 #endif TSAN_INTERCEPTOR(int, creat, const char *name, int mode) { SCOPED_TSAN_INTERCEPTOR(creat, name, mode); READ_STRING(thr, pc, name, 0); int fd = REAL(creat)(name, mode); if (fd >= 0) FdFileCreate(thr, pc, fd); return fd; } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) { SCOPED_TSAN_INTERCEPTOR(creat64, name, mode); READ_STRING(thr, pc, name, 0); int fd = REAL(creat64)(name, mode); if (fd >= 0) FdFileCreate(thr, pc, fd); return fd; } #define TSAN_MAYBE_INTERCEPT_CREAT64 TSAN_INTERCEPT(creat64) #else #define TSAN_MAYBE_INTERCEPT_CREAT64 #endif TSAN_INTERCEPTOR(int, dup, int oldfd) { SCOPED_TSAN_INTERCEPTOR(dup, oldfd); int newfd = REAL(dup)(oldfd); if (oldfd >= 0 && newfd >= 0 && newfd != oldfd) FdDup(thr, pc, oldfd, newfd, true); return newfd; } TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) { SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd); int newfd2 = REAL(dup2)(oldfd, newfd); if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) FdDup(thr, pc, oldfd, newfd2, false); return newfd2; } #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) { SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags); int newfd2 = REAL(dup3)(oldfd, newfd, flags); if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) FdDup(thr, pc, oldfd, newfd2, false); return newfd2; } #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) { SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags); int fd = REAL(eventfd)(initval, flags); if (fd >= 0) FdEventCreate(thr, pc, fd); return fd; } #define TSAN_MAYBE_INTERCEPT_EVENTFD TSAN_INTERCEPT(eventfd) #else #define TSAN_MAYBE_INTERCEPT_EVENTFD #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) { SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags); if (fd >= 0) FdClose(thr, pc, fd); fd = REAL(signalfd)(fd, mask, flags); if (fd >= 0) FdSignalCreate(thr, pc, fd); return fd; } #define TSAN_MAYBE_INTERCEPT_SIGNALFD TSAN_INTERCEPT(signalfd) #else #define TSAN_MAYBE_INTERCEPT_SIGNALFD #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, inotify_init, int fake) { SCOPED_TSAN_INTERCEPTOR(inotify_init, fake); int fd = REAL(inotify_init)(fake); if (fd >= 0) FdInotifyCreate(thr, pc, fd); return fd; } #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT TSAN_INTERCEPT(inotify_init) #else #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, inotify_init1, int flags) { SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags); int fd = REAL(inotify_init1)(flags); if (fd >= 0) FdInotifyCreate(thr, pc, fd); return fd; } #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 TSAN_INTERCEPT(inotify_init1) #else #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 #endif TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) { SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol); int fd = REAL(socket)(domain, type, protocol); if (fd >= 0) FdSocketCreate(thr, pc, fd); return fd; } TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) { SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd); int res = REAL(socketpair)(domain, type, protocol, fd); if (res == 0 && fd[0] >= 0 && fd[1] >= 0) FdPipeCreate(thr, pc, fd[0], fd[1]); return res; } TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) { SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen); FdSocketConnecting(thr, pc, fd); int res = REAL(connect)(fd, addr, addrlen); if (res == 0 && fd >= 0) FdSocketConnect(thr, pc, fd); return res; } TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) { SCOPED_TSAN_INTERCEPTOR(bind, fd, addr, addrlen); int res = REAL(bind)(fd, addr, addrlen); if (fd > 0 && res == 0) FdAccess(thr, pc, fd); return res; } TSAN_INTERCEPTOR(int, listen, int fd, int backlog) { SCOPED_TSAN_INTERCEPTOR(listen, fd, backlog); int res = REAL(listen)(fd, backlog); if (fd > 0 && res == 0) FdAccess(thr, pc, fd); return res; } TSAN_INTERCEPTOR(int, close, int fd) { SCOPED_TSAN_INTERCEPTOR(close, fd); if (fd >= 0) FdClose(thr, pc, fd); return REAL(close)(fd); } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, __close, int fd) { SCOPED_TSAN_INTERCEPTOR(__close, fd); if (fd >= 0) FdClose(thr, pc, fd); return REAL(__close)(fd); } #define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close) #else #define TSAN_MAYBE_INTERCEPT___CLOSE #endif // glibc guts #if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) { SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr); int fds[64]; int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds)); for (int i = 0; i < cnt; i++) { if (fds[i] > 0) FdClose(thr, pc, fds[i]); } REAL(__res_iclose)(state, free_addr); } #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose) #else #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE #endif TSAN_INTERCEPTOR(int, pipe, int *pipefd) { SCOPED_TSAN_INTERCEPTOR(pipe, pipefd); int res = REAL(pipe)(pipefd); if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); return res; } #if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) { SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags); int res = REAL(pipe2)(pipefd, flags); if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); return res; } #endif TSAN_INTERCEPTOR(int, unlink, char *path) { SCOPED_TSAN_INTERCEPTOR(unlink, path); Release(thr, pc, File2addr(path)); int res = REAL(unlink)(path); return res; } TSAN_INTERCEPTOR(void*, tmpfile, int fake) { SCOPED_TSAN_INTERCEPTOR(tmpfile, fake); void *res = REAL(tmpfile)(fake); if (res) { int fd = fileno_unlocked(res); if (fd >= 0) FdFileCreate(thr, pc, fd); } return res; } #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, tmpfile64, int fake) { SCOPED_TSAN_INTERCEPTOR(tmpfile64, fake); void *res = REAL(tmpfile64)(fake); if (res) { int fd = fileno_unlocked(res); if (fd >= 0) FdFileCreate(thr, pc, fd); } return res; } #define TSAN_MAYBE_INTERCEPT_TMPFILE64 TSAN_INTERCEPT(tmpfile64) #else #define TSAN_MAYBE_INTERCEPT_TMPFILE64 #endif TSAN_INTERCEPTOR(uptr, fread, void *ptr, uptr size, uptr nmemb, void *f) { // libc file streams can call user-supplied functions, see fopencookie. { SCOPED_TSAN_INTERCEPTOR(fread, ptr, size, nmemb, f); MemoryAccessRange(thr, pc, (uptr)ptr, size * nmemb, true); } return REAL(fread)(ptr, size, nmemb, f); } TSAN_INTERCEPTOR(uptr, fwrite, const void *p, uptr size, uptr nmemb, void *f) { // libc file streams can call user-supplied functions, see fopencookie. { SCOPED_TSAN_INTERCEPTOR(fwrite, p, size, nmemb, f); MemoryAccessRange(thr, pc, (uptr)p, size * nmemb, false); } return REAL(fwrite)(p, size, nmemb, f); } static void FlushStreams() { // Flushing all the streams here may freeze the process if a child thread is // performing file stream operations at the same time. REAL(fflush)(stdout); REAL(fflush)(stderr); } TSAN_INTERCEPTOR(void, abort, int fake) { SCOPED_TSAN_INTERCEPTOR(abort, fake); FlushStreams(); REAL(abort)(fake); } TSAN_INTERCEPTOR(int, puts, const char *s) { SCOPED_TSAN_INTERCEPTOR(puts, s); MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s), false); return REAL(puts)(s); } TSAN_INTERCEPTOR(int, rmdir, char *path) { SCOPED_TSAN_INTERCEPTOR(rmdir, path); Release(thr, pc, Dir2addr(path)); int res = REAL(rmdir)(path); return res; } TSAN_INTERCEPTOR(int, closedir, void *dirp) { SCOPED_TSAN_INTERCEPTOR(closedir, dirp); if (dirp) { int fd = dirfd(dirp); FdClose(thr, pc, fd); } return REAL(closedir)(dirp); } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, epoll_create, int size) { SCOPED_TSAN_INTERCEPTOR(epoll_create, size); int fd = REAL(epoll_create)(size); if (fd >= 0) FdPollCreate(thr, pc, fd); return fd; } TSAN_INTERCEPTOR(int, epoll_create1, int flags) { SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags); int fd = REAL(epoll_create1)(flags); if (fd >= 0) FdPollCreate(thr, pc, fd); return fd; } TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) { SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev); if (epfd >= 0) FdAccess(thr, pc, epfd); if (epfd >= 0 && fd >= 0) FdAccess(thr, pc, fd); if (op == EPOLL_CTL_ADD && epfd >= 0) FdRelease(thr, pc, epfd); int res = REAL(epoll_ctl)(epfd, op, fd, ev); return res; } TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout); if (epfd >= 0) FdAccess(thr, pc, epfd); int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout); if (res > 0 && epfd >= 0) FdAcquire(thr, pc, epfd); return res; } TSAN_INTERCEPTOR(int, epoll_pwait, int epfd, void *ev, int cnt, int timeout, void *sigmask) { SCOPED_TSAN_INTERCEPTOR(epoll_pwait, epfd, ev, cnt, timeout, sigmask); if (epfd >= 0) FdAccess(thr, pc, epfd); int res = BLOCK_REAL(epoll_pwait)(epfd, ev, cnt, timeout, sigmask); if (res > 0 && epfd >= 0) FdAcquire(thr, pc, epfd); return res; } #define TSAN_MAYBE_INTERCEPT_EPOLL \ TSAN_INTERCEPT(epoll_create); \ TSAN_INTERCEPT(epoll_create1); \ TSAN_INTERCEPT(epoll_ctl); \ TSAN_INTERCEPT(epoll_wait); \ TSAN_INTERCEPT(epoll_pwait) #else #define TSAN_MAYBE_INTERCEPT_EPOLL #endif // The following functions are intercepted merely to process pending signals. // If program blocks signal X, we must deliver the signal before the function // returns. Similarly, if program unblocks a signal (or returns from sigsuspend) // it's better to deliver the signal straight away. TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) { SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask); return REAL(sigsuspend)(mask); } TSAN_INTERCEPTOR(int, sigblock, int mask) { SCOPED_TSAN_INTERCEPTOR(sigblock, mask); return REAL(sigblock)(mask); } TSAN_INTERCEPTOR(int, sigsetmask, int mask) { SCOPED_TSAN_INTERCEPTOR(sigsetmask, mask); return REAL(sigsetmask)(mask); } TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { SCOPED_TSAN_INTERCEPTOR(pthread_sigmask, how, set, oldset); return REAL(pthread_sigmask)(how, set, oldset); } namespace __tsan { static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, bool sigact, int sig, my_siginfo_t *info, void *uctx) { if (acquire) Acquire(thr, 0, (uptr)&sigactions[sig]); // Signals are generally asynchronous, so if we receive a signals when // ignores are enabled we should disable ignores. This is critical for sync // and interceptors, because otherwise we can miss syncronization and report // false races. int ignore_reads_and_writes = thr->ignore_reads_and_writes; int ignore_interceptors = thr->ignore_interceptors; int ignore_sync = thr->ignore_sync; if (!ctx->after_multithreaded_fork) { thr->ignore_reads_and_writes = 0; thr->fast_state.ClearIgnoreBit(); thr->ignore_interceptors = 0; thr->ignore_sync = 0; } // Ensure that the handler does not spoil errno. const int saved_errno = errno; errno = 99; // This code races with sigaction. Be careful to not read sa_sigaction twice. // Also need to remember pc for reporting before the call, // because the handler can reset it. volatile uptr pc = sigact ? (uptr)sigactions[sig].sa_sigaction : (uptr)sigactions[sig].sa_handler; if (pc != (uptr)SIG_DFL && pc != (uptr)SIG_IGN) { if (sigact) ((sigactionhandler_t)pc)(sig, info, uctx); else ((sighandler_t)pc)(sig); } if (!ctx->after_multithreaded_fork) { thr->ignore_reads_and_writes = ignore_reads_and_writes; if (ignore_reads_and_writes) thr->fast_state.SetIgnoreBit(); thr->ignore_interceptors = ignore_interceptors; thr->ignore_sync = ignore_sync; } // We do not detect errno spoiling for SIGTERM, // because some SIGTERM handlers do spoil errno but reraise SIGTERM, // tsan reports false positive in such case. // It's difficult to properly detect this situation (reraise), // because in async signal processing case (when handler is called directly // from rtl_generic_sighandler) we have not yet received the reraised // signal; and it looks too fragile to intercept all ways to reraise a signal. if (flags()->report_bugs && !sync && sig != SIGTERM && errno != 99) { VarSizeStackTrace stack; // StackTrace::GetNestInstructionPc(pc) is used because return address is // expected, OutputReport() will undo this. ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack); ThreadRegistryLock l(ctx->thread_registry); ScopedReport rep(ReportTypeErrnoInSignal); if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) { rep.AddStack(stack, true); OutputReport(thr, rep); } } errno = saved_errno; } void ProcessPendingSignals(ThreadState *thr) { ThreadSignalContext *sctx = SigCtx(thr); if (sctx == 0 || atomic_load(&sctx->have_pending_signals, memory_order_relaxed) == 0) return; atomic_store(&sctx->have_pending_signals, 0, memory_order_relaxed); atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); internal_sigfillset(&sctx->emptyset); int res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->emptyset, &sctx->oldset); CHECK_EQ(res, 0); for (int sig = 0; sig < kSigCount; sig++) { SignalDesc *signal = &sctx->pending_signals[sig]; if (signal->armed) { signal->armed = false; CallUserSignalHandler(thr, false, true, signal->sigaction, sig, &signal->siginfo, &signal->ctx); } } res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->oldset, 0); CHECK_EQ(res, 0); atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); } } // namespace __tsan static bool is_sync_signal(ThreadSignalContext *sctx, int sig) { return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS || // If we are sending signal to ourselves, we must process it now. (sctx && sig == sctx->int_signal_send); } void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, my_siginfo_t *info, void *ctx) { ThreadState *thr = cur_thread(); ThreadSignalContext *sctx = SigCtx(thr); if (sig < 0 || sig >= kSigCount) { VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig); return; } // Don't mess with synchronous signals. const bool sync = is_sync_signal(sctx, sig); if (sync || // If we are in blocking function, we can safely process it now // (but check if we are in a recursive interceptor, // i.e. pthread_join()->munmap()). (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed))) { atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); if (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed)) { atomic_store(&sctx->in_blocking_func, 0, memory_order_relaxed); CallUserSignalHandler(thr, sync, true, sigact, sig, info, ctx); atomic_store(&sctx->in_blocking_func, 1, memory_order_relaxed); } else { // Be very conservative with when we do acquire in this case. // It's unsafe to do acquire in async handlers, because ThreadState // can be in inconsistent state. // SIGSYS looks relatively safe -- it's synchronous and can actually // need some global state. bool acq = (sig == SIGSYS); CallUserSignalHandler(thr, sync, acq, sigact, sig, info, ctx); } atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); return; } if (sctx == 0) return; SignalDesc *signal = &sctx->pending_signals[sig]; if (signal->armed == false) { signal->armed = true; signal->sigaction = sigact; if (info) internal_memcpy(&signal->siginfo, info, sizeof(*info)); if (ctx) internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx)); atomic_store(&sctx->have_pending_signals, 1, memory_order_relaxed); } } static void rtl_sighandler(int sig) { rtl_generic_sighandler(false, sig, 0, 0); } static void rtl_sigaction(int sig, my_siginfo_t *info, void *ctx) { rtl_generic_sighandler(true, sig, info, ctx); } TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) { // Note: if we call REAL(sigaction) directly for any reason without proxying // the signal handler through rtl_sigaction, very bad things will happen. // The handler will run synchronously and corrupt tsan per-thread state. SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old); if (old) internal_memcpy(old, &sigactions[sig], sizeof(*old)); if (act == 0) return 0; // Copy act into sigactions[sig]. // Can't use struct copy, because compiler can emit call to memcpy. // Can't use internal_memcpy, because it copies byte-by-byte, // and signal handler reads the sa_handler concurrently. It it can read // some bytes from old value and some bytes from new value. // Use volatile to prevent insertion of memcpy. sigactions[sig].sa_handler = *(volatile sighandler_t*)&act->sa_handler; sigactions[sig].sa_flags = *(volatile int*)&act->sa_flags; internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask, sizeof(sigactions[sig].sa_mask)); #if !SANITIZER_FREEBSD && !SANITIZER_MAC sigactions[sig].sa_restorer = act->sa_restorer; #endif sigaction_t newact; internal_memcpy(&newact, act, sizeof(newact)); internal_sigfillset(&newact.sa_mask); if (act->sa_handler != SIG_IGN && act->sa_handler != SIG_DFL) { if (newact.sa_flags & SA_SIGINFO) newact.sa_sigaction = rtl_sigaction; else newact.sa_handler = rtl_sighandler; } ReleaseStore(thr, pc, (uptr)&sigactions[sig]); int res = REAL(sigaction)(sig, &newact, 0); return res; } TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) { sigaction_t act; act.sa_handler = h; internal_memset(&act.sa_mask, -1, sizeof(act.sa_mask)); act.sa_flags = 0; sigaction_t old; int res = sigaction(sig, &act, &old); if (res) return SIG_ERR; return old.sa_handler; } TSAN_INTERCEPTOR(int, raise, int sig) { SCOPED_TSAN_INTERCEPTOR(raise, sig); ThreadSignalContext *sctx = SigCtx(thr); CHECK_NE(sctx, 0); int prev = sctx->int_signal_send; sctx->int_signal_send = sig; int res = REAL(raise)(sig); CHECK_EQ(sctx->int_signal_send, sig); sctx->int_signal_send = prev; return res; } TSAN_INTERCEPTOR(int, kill, int pid, int sig) { SCOPED_TSAN_INTERCEPTOR(kill, pid, sig); ThreadSignalContext *sctx = SigCtx(thr); CHECK_NE(sctx, 0); int prev = sctx->int_signal_send; if (pid == (int)internal_getpid()) { sctx->int_signal_send = sig; } int res = REAL(kill)(pid, sig); if (pid == (int)internal_getpid()) { CHECK_EQ(sctx->int_signal_send, sig); sctx->int_signal_send = prev; } return res; } TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) { SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig); ThreadSignalContext *sctx = SigCtx(thr); CHECK_NE(sctx, 0); int prev = sctx->int_signal_send; if (tid == pthread_self()) { sctx->int_signal_send = sig; } int res = REAL(pthread_kill)(tid, sig); if (tid == pthread_self()) { CHECK_EQ(sctx->int_signal_send, sig); sctx->int_signal_send = prev; } return res; } TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) { SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz); // It's intercepted merely to process pending signals. return REAL(gettimeofday)(tv, tz); } TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, void *hints, void *rv) { SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv); // We miss atomic synchronization in getaddrinfo, // and can report false race between malloc and free // inside of getaddrinfo. So ignore memory accesses. ThreadIgnoreBegin(thr, pc); int res = REAL(getaddrinfo)(node, service, hints, rv); ThreadIgnoreEnd(thr, pc); return res; } TSAN_INTERCEPTOR(int, fork, int fake) { if (cur_thread()->in_symbolizer) return REAL(fork)(fake); SCOPED_INTERCEPTOR_RAW(fork, fake); ForkBefore(thr, pc); int pid; { // On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and // we'll assert in CheckNoLocks() unless we ignore interceptors. ScopedIgnoreInterceptors ignore; pid = REAL(fork)(fake); } if (pid == 0) { // child ForkChildAfter(thr, pc); FdOnFork(thr, pc); } else if (pid > 0) { // parent ForkParentAfter(thr, pc); } else { // error ForkParentAfter(thr, pc); } return pid; } TSAN_INTERCEPTOR(int, vfork, int fake) { // Some programs (e.g. openjdk) call close for all file descriptors // in the child process. Under tsan it leads to false positives, because // address space is shared, so the parent process also thinks that // the descriptors are closed (while they are actually not). // This leads to false positives due to missed synchronization. // Strictly saying this is undefined behavior, because vfork child is not // allowed to call any functions other than exec/exit. But this is what // openjdk does, so we want to handle it. // We could disable interceptors in the child process. But it's not possible // to simply intercept and wrap vfork, because vfork child is not allowed // to return from the function that calls vfork, and that's exactly what // we would do. So this would require some assembly trickery as well. // Instead we simply turn vfork into fork. return WRAP(fork)(fake); } #if !SANITIZER_MAC && !SANITIZER_ANDROID typedef int (*dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info *info, SIZE_T size, void *data); struct dl_iterate_phdr_data { ThreadState *thr; uptr pc; dl_iterate_phdr_cb_t cb; void *data; }; static bool IsAppNotRodata(uptr addr) { return IsAppMem(addr) && *(u64*)MemToShadow(addr) != kShadowRodata; } static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size, void *data) { dl_iterate_phdr_data *cbdata = (dl_iterate_phdr_data *)data; // dlopen/dlclose allocate/free dynamic-linker-internal memory, which is later // accessible in dl_iterate_phdr callback. But we don't see synchronization // inside of dynamic linker, so we "unpoison" it here in order to not // produce false reports. Ignoring malloc/free in dlopen/dlclose is not enough // because some libc functions call __libc_dlopen. if (info && IsAppNotRodata((uptr)info->dlpi_name)) MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name, internal_strlen(info->dlpi_name)); int res = cbdata->cb(info, size, cbdata->data); // Perform the check one more time in case info->dlpi_name was overwritten // by user callback. if (info && IsAppNotRodata((uptr)info->dlpi_name)) MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name, internal_strlen(info->dlpi_name)); return res; } TSAN_INTERCEPTOR(int, dl_iterate_phdr, dl_iterate_phdr_cb_t cb, void *data) { SCOPED_TSAN_INTERCEPTOR(dl_iterate_phdr, cb, data); dl_iterate_phdr_data cbdata; cbdata.thr = thr; cbdata.pc = pc; cbdata.cb = cb; cbdata.data = data; int res = REAL(dl_iterate_phdr)(dl_iterate_phdr_cb, &cbdata); return res; } #endif static int OnExit(ThreadState *thr) { int status = Finalize(thr); FlushStreams(); return status; } struct TsanInterceptorContext { ThreadState *thr; const uptr caller_pc; const uptr pc; }; #if !SANITIZER_MAC static void HandleRecvmsg(ThreadState *thr, uptr pc, __sanitizer_msghdr *msg) { int fds[64]; int cnt = ExtractRecvmsgFDs(msg, fds, ARRAY_SIZE(fds)); for (int i = 0; i < cnt; i++) FdEventCreate(thr, pc, fds[i]); } #endif #include "sanitizer_common/sanitizer_platform_interceptors.h" // Causes interceptor recursion (getaddrinfo() and fopen()) #undef SANITIZER_INTERCEPT_GETADDRINFO // There interceptors do not seem to be strictly necessary for tsan. // But we see cases where the interceptors consume 70% of execution time. // Memory blocks passed to fgetgrent_r are "written to" by tsan several times. // First, there is some recursion (getgrnam_r calls fgetgrent_r), and each // function "writes to" the buffer. Then, the same memory is "written to" // twice, first as buf and then as pwbufp (both of them refer to the same // addresses). #undef SANITIZER_INTERCEPT_GETPWENT #undef SANITIZER_INTERCEPT_GETPWENT_R #undef SANITIZER_INTERCEPT_FGETPWENT #undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS #undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS // We define our own. #if SANITIZER_INTERCEPT_TLS_GET_ADDR #define NEED_TLS_GET_ADDR #endif #undef SANITIZER_INTERCEPT_TLS_GET_ADDR #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ INTERCEPT_FUNCTION_VER(name, ver) #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \ true) #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \ ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \ false) #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \ TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \ ctx = (void *)&_ctx; \ (void) ctx; #define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \ SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \ ctx = (void *)&_ctx; \ (void) ctx; #define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \ Acquire(thr, pc, File2addr(path)); \ if (file) { \ int fd = fileno_unlocked(file); \ if (fd >= 0) FdFileCreate(thr, pc, fd); \ } #define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \ if (file) { \ int fd = fileno_unlocked(file); \ if (fd >= 0) FdClose(thr, pc, fd); \ } #define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \ libignore()->OnLibraryLoaded(filename) #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \ libignore()->OnLibraryUnloaded() #define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \ Acquire(((TsanInterceptorContext *) ctx)->thr, pc, u) #define COMMON_INTERCEPTOR_RELEASE(ctx, u) \ Release(((TsanInterceptorContext *) ctx)->thr, pc, u) #define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ Acquire(((TsanInterceptorContext *) ctx)->thr, pc, Dir2addr(path)) #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd) #define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd) #define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \ FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd) #define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd) #define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name) #define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ __tsan::ctx->thread_registry->SetThreadNameByUserId(thread, name) #define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name) #define COMMON_INTERCEPTOR_ON_EXIT(ctx) \ OnExit(((TsanInterceptorContext *) ctx)->thr) #define COMMON_INTERCEPTOR_MUTEX_LOCK(ctx, m) \ MutexLock(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, (uptr)m) #define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \ MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, (uptr)m) #define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \ MutexRepair(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, (uptr)m) #define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) \ MutexInvalidAccess(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, (uptr)m) #if !SANITIZER_MAC #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \ HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, msg) #endif #define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ if (TsanThread *t = GetCurrentThread()) { \ *begin = t->tls_begin(); \ *end = t->tls_end(); \ } else { \ *begin = *end = 0; \ } #define COMMON_INTERCEPTOR_USER_CALLBACK_START() \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() #define COMMON_INTERCEPTOR_USER_CALLBACK_END() \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END() #include "sanitizer_common/sanitizer_common_interceptors.inc" #define TSAN_SYSCALL() \ ThreadState *thr = cur_thread(); \ if (thr->ignore_interceptors) \ return; \ ScopedSyscall scoped_syscall(thr) \ /**/ struct ScopedSyscall { ThreadState *thr; explicit ScopedSyscall(ThreadState *thr) : thr(thr) { Initialize(thr); } ~ScopedSyscall() { ProcessPendingSignals(thr); } }; #if !SANITIZER_FREEBSD && !SANITIZER_MAC static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { TSAN_SYSCALL(); MemoryAccessRange(thr, pc, p, s, write); } static void syscall_acquire(uptr pc, uptr addr) { TSAN_SYSCALL(); Acquire(thr, pc, addr); DPrintf("syscall_acquire(%p)\n", addr); } static void syscall_release(uptr pc, uptr addr) { TSAN_SYSCALL(); DPrintf("syscall_release(%p)\n", addr); Release(thr, pc, addr); } static void syscall_fd_close(uptr pc, int fd) { TSAN_SYSCALL(); FdClose(thr, pc, fd); } static USED void syscall_fd_acquire(uptr pc, int fd) { TSAN_SYSCALL(); FdAcquire(thr, pc, fd); DPrintf("syscall_fd_acquire(%p)\n", fd); } static USED void syscall_fd_release(uptr pc, int fd) { TSAN_SYSCALL(); DPrintf("syscall_fd_release(%p)\n", fd); FdRelease(thr, pc, fd); } static void syscall_pre_fork(uptr pc) { TSAN_SYSCALL(); ForkBefore(thr, pc); } static void syscall_post_fork(uptr pc, int pid) { TSAN_SYSCALL(); if (pid == 0) { // child ForkChildAfter(thr, pc); FdOnFork(thr, pc); } else if (pid > 0) { // parent ForkParentAfter(thr, pc); } else { // error ForkParentAfter(thr, pc); } } #endif #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \ syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false) #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true) #define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ do { \ (void)(p); \ (void)(s); \ } while (false) #define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ do { \ (void)(p); \ (void)(s); \ } while (false) #define COMMON_SYSCALL_ACQUIRE(addr) \ syscall_acquire(GET_CALLER_PC(), (uptr)(addr)) #define COMMON_SYSCALL_RELEASE(addr) \ syscall_release(GET_CALLER_PC(), (uptr)(addr)) #define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd) #define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd) #define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd) #define COMMON_SYSCALL_PRE_FORK() \ syscall_pre_fork(GET_CALLER_PC()) #define COMMON_SYSCALL_POST_FORK(res) \ syscall_post_fork(GET_CALLER_PC(), res) #include "sanitizer_common/sanitizer_common_syscalls.inc" #ifdef NEED_TLS_GET_ADDR // Define own interceptor instead of sanitizer_common's for three reasons: // 1. It must not process pending signals. // Signal handlers may contain MOVDQA instruction (see below). // 2. It must be as simple as possible to not contain MOVDQA. // 3. Sanitizer_common version uses COMMON_INTERCEPTOR_INITIALIZE_RANGE which // is empty for tsan (meant only for msan). // Note: __tls_get_addr can be called with mis-aligned stack due to: // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066 // So the interceptor must work with mis-aligned stack, in particular, does not // execute MOVDQA with stack addresses. TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) { void *res = REAL(__tls_get_addr)(arg); ThreadState *thr = cur_thread(); if (!thr) return res; DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr, thr->tls_size); if (!dtv) return res; // New DTLS block has been allocated. MemoryResetRange(thr, 0, dtv->beg, dtv->size); return res; } #endif namespace __tsan { static void finalize(void *arg) { ThreadState *thr = cur_thread(); int status = Finalize(thr); // Make sure the output is not lost. FlushStreams(); if (status) Die(); } #if !SANITIZER_MAC && !SANITIZER_ANDROID static void unreachable() { Report("FATAL: ThreadSanitizer: unreachable called\n"); Die(); } #endif void InitializeInterceptors() { #if !SANITIZER_MAC // We need to setup it early, because functions like dlsym() can call it. REAL(memset) = internal_memset; REAL(memcpy) = internal_memcpy; #endif // Instruct libc malloc to consume less memory. #if SANITIZER_LINUX mallopt(1, 0); // M_MXFAST mallopt(-3, 32*1024); // M_MMAP_THRESHOLD #endif InitializeCommonInterceptors(); #if !SANITIZER_MAC // We can not use TSAN_INTERCEPT to get setjmp addr, // because it does &setjmp and setjmp is not present in some versions of libc. using __interception::GetRealFunctionAddress; GetRealFunctionAddress("setjmp", (uptr*)&REAL(setjmp), 0, 0); GetRealFunctionAddress("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); GetRealFunctionAddress("sigsetjmp", (uptr*)&REAL(sigsetjmp), 0, 0); GetRealFunctionAddress("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); #endif TSAN_INTERCEPT(longjmp); TSAN_INTERCEPT(siglongjmp); TSAN_INTERCEPT(malloc); TSAN_INTERCEPT(__libc_memalign); TSAN_INTERCEPT(calloc); TSAN_INTERCEPT(realloc); TSAN_INTERCEPT(free); TSAN_INTERCEPT(cfree); TSAN_INTERCEPT(mmap); TSAN_MAYBE_INTERCEPT_MMAP64; TSAN_INTERCEPT(munmap); TSAN_MAYBE_INTERCEPT_MEMALIGN; TSAN_INTERCEPT(valloc); TSAN_MAYBE_INTERCEPT_PVALLOC; TSAN_INTERCEPT(posix_memalign); TSAN_INTERCEPT(strcpy); // NOLINT TSAN_INTERCEPT(strncpy); TSAN_INTERCEPT(strdup); TSAN_INTERCEPT(pthread_create); TSAN_INTERCEPT(pthread_join); TSAN_INTERCEPT(pthread_detach); TSAN_INTERCEPT_VER(pthread_cond_init, PTHREAD_ABI_BASE); TSAN_INTERCEPT_VER(pthread_cond_signal, PTHREAD_ABI_BASE); TSAN_INTERCEPT_VER(pthread_cond_broadcast, PTHREAD_ABI_BASE); TSAN_INTERCEPT_VER(pthread_cond_wait, PTHREAD_ABI_BASE); TSAN_INTERCEPT_VER(pthread_cond_timedwait, PTHREAD_ABI_BASE); TSAN_INTERCEPT_VER(pthread_cond_destroy, PTHREAD_ABI_BASE); TSAN_INTERCEPT(pthread_mutex_init); TSAN_INTERCEPT(pthread_mutex_destroy); TSAN_INTERCEPT(pthread_mutex_trylock); TSAN_INTERCEPT(pthread_mutex_timedlock); TSAN_INTERCEPT(pthread_spin_init); TSAN_INTERCEPT(pthread_spin_destroy); TSAN_INTERCEPT(pthread_spin_lock); TSAN_INTERCEPT(pthread_spin_trylock); TSAN_INTERCEPT(pthread_spin_unlock); TSAN_INTERCEPT(pthread_rwlock_init); TSAN_INTERCEPT(pthread_rwlock_destroy); TSAN_INTERCEPT(pthread_rwlock_rdlock); TSAN_INTERCEPT(pthread_rwlock_tryrdlock); TSAN_INTERCEPT(pthread_rwlock_timedrdlock); TSAN_INTERCEPT(pthread_rwlock_wrlock); TSAN_INTERCEPT(pthread_rwlock_trywrlock); TSAN_INTERCEPT(pthread_rwlock_timedwrlock); TSAN_INTERCEPT(pthread_rwlock_unlock); TSAN_INTERCEPT(pthread_barrier_init); TSAN_INTERCEPT(pthread_barrier_destroy); TSAN_INTERCEPT(pthread_barrier_wait); TSAN_INTERCEPT(pthread_once); TSAN_INTERCEPT(fstat); TSAN_MAYBE_INTERCEPT___FXSTAT; TSAN_MAYBE_INTERCEPT_FSTAT64; TSAN_MAYBE_INTERCEPT___FXSTAT64; TSAN_INTERCEPT(open); TSAN_MAYBE_INTERCEPT_OPEN64; TSAN_INTERCEPT(creat); TSAN_MAYBE_INTERCEPT_CREAT64; TSAN_INTERCEPT(dup); TSAN_INTERCEPT(dup2); TSAN_INTERCEPT(dup3); TSAN_MAYBE_INTERCEPT_EVENTFD; TSAN_MAYBE_INTERCEPT_SIGNALFD; TSAN_MAYBE_INTERCEPT_INOTIFY_INIT; TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1; TSAN_INTERCEPT(socket); TSAN_INTERCEPT(socketpair); TSAN_INTERCEPT(connect); TSAN_INTERCEPT(bind); TSAN_INTERCEPT(listen); TSAN_MAYBE_INTERCEPT_EPOLL; TSAN_INTERCEPT(close); TSAN_MAYBE_INTERCEPT___CLOSE; TSAN_MAYBE_INTERCEPT___RES_ICLOSE; TSAN_INTERCEPT(pipe); TSAN_INTERCEPT(pipe2); TSAN_INTERCEPT(unlink); TSAN_INTERCEPT(tmpfile); TSAN_MAYBE_INTERCEPT_TMPFILE64; TSAN_INTERCEPT(fread); TSAN_INTERCEPT(fwrite); TSAN_INTERCEPT(abort); TSAN_INTERCEPT(puts); TSAN_INTERCEPT(rmdir); TSAN_INTERCEPT(closedir); TSAN_INTERCEPT(sigaction); TSAN_INTERCEPT(signal); TSAN_INTERCEPT(sigsuspend); TSAN_INTERCEPT(sigblock); TSAN_INTERCEPT(sigsetmask); TSAN_INTERCEPT(pthread_sigmask); TSAN_INTERCEPT(raise); TSAN_INTERCEPT(kill); TSAN_INTERCEPT(pthread_kill); TSAN_INTERCEPT(sleep); TSAN_INTERCEPT(usleep); TSAN_INTERCEPT(nanosleep); TSAN_INTERCEPT(gettimeofday); TSAN_INTERCEPT(getaddrinfo); TSAN_INTERCEPT(fork); TSAN_INTERCEPT(vfork); #if !SANITIZER_ANDROID TSAN_INTERCEPT(dl_iterate_phdr); #endif TSAN_INTERCEPT(on_exit); TSAN_INTERCEPT(__cxa_atexit); TSAN_INTERCEPT(_exit); #ifdef NEED_TLS_GET_ADDR TSAN_INTERCEPT(__tls_get_addr); #endif #if !SANITIZER_MAC && !SANITIZER_ANDROID // Need to setup it, because interceptors check that the function is resolved. // But atexit is emitted directly into the module, so can't be resolved. REAL(atexit) = (int(*)(void(*)()))unreachable; #endif if (REAL(__cxa_atexit)(&finalize, 0, 0)) { Printf("ThreadSanitizer: failed to setup atexit callback\n"); Die(); } #if !SANITIZER_MAC if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) { Printf("ThreadSanitizer: failed to create thread key\n"); Die(); } #endif FdInit(); } } // namespace __tsan // Invisible barrier for tests. // There were several unsuccessful iterations for this functionality: // 1. Initially it was implemented in user code using // REAL(pthread_barrier_wait). But pthread_barrier_wait is not supported on // MacOS. Futexes are linux-specific for this matter. // 2. Then we switched to atomics+usleep(10). But usleep produced parasitic // "as-if synchronized via sleep" messages in reports which failed some // output tests. // 3. Then we switched to atomics+sched_yield. But this produced tons of tsan- // visible events, which lead to "failed to restore stack trace" failures. // Note that no_sanitize_thread attribute does not turn off atomic interception // so attaching it to the function defined in user code does not help. // That's why we now have what we have. extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_init(u64 *barrier, u32 count) { if (count >= (1 << 8)) { Printf("barrier_init: count is too large (%d)\n", count); Die(); } // 8 lsb is thread count, the remaining are count of entered threads. *barrier = count; } extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait(u64 *barrier) { unsigned old = __atomic_fetch_add(barrier, 1 << 8, __ATOMIC_RELAXED); unsigned old_epoch = (old >> 8) / (old & 0xff); for (;;) { unsigned cur = __atomic_load_n(barrier, __ATOMIC_RELAXED); unsigned cur_epoch = (cur >> 8) / (cur & 0xff); if (cur_epoch != old_epoch) return; internal_sched_yield(); } } Index: projects/clang400-import/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/tsan/rtl/tsan_interceptors.h (revision 312198) @@ -1,49 +1,50 @@ #ifndef TSAN_INTERCEPTORS_H #define TSAN_INTERCEPTORS_H #include "sanitizer_common/sanitizer_stacktrace.h" #include "tsan_rtl.h" namespace __tsan { class ScopedInterceptor { public: ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc); ~ScopedInterceptor(); - void UserCallbackStart(); - void UserCallbackEnd(); + void DisableIgnores(); + void EnableIgnores(); private: ThreadState *const thr_; const uptr pc_; bool in_ignored_lib_; + bool ignoring_; }; } // namespace __tsan #define SCOPED_INTERCEPTOR_RAW(func, ...) \ ThreadState *thr = cur_thread(); \ const uptr caller_pc = GET_CALLER_PC(); \ ScopedInterceptor si(thr, #func, caller_pc); \ const uptr pc = StackTrace::GetCurrentPc(); \ (void)pc; \ /**/ #define SCOPED_TSAN_INTERCEPTOR(func, ...) \ SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ if (REAL(func) == 0) { \ Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \ Die(); \ } \ if (!thr->is_inited || thr->ignore_interceptors || thr->in_ignored_lib) \ return REAL(func)(__VA_ARGS__); \ /**/ #define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() \ - si.UserCallbackStart(); + si.DisableIgnores(); #define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END() \ - si.UserCallbackEnd(); + si.EnableIgnores(); #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) #endif // TSAN_INTERCEPTORS_H Index: projects/clang400-import/contrib/compiler-rt/lib/xray/xray_AArch64.cc =================================================================== --- projects/clang400-import/contrib/compiler-rt/lib/xray/xray_AArch64.cc (revision 312197) +++ projects/clang400-import/contrib/compiler-rt/lib/xray/xray_AArch64.cc (revision 312198) @@ -1,119 +1,125 @@ //===-- xray_AArch64.cc -----------------------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// // // This file is a part of XRay, a dynamic runtime instrumentation system. // // Implementation of AArch64-specific routines (64-bit). // //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_common.h" #include "xray_defs.h" #include "xray_emulate_tsc.h" #include "xray_interface_internal.h" #include #include + +extern "C" void __clear_cache(void* start, void* end); + namespace __xray { uint64_t cycleFrequency() XRAY_NEVER_INSTRUMENT { // There is no instruction like RDTSCP in user mode on ARM. ARM's CP15 does // not have a constant frequency like TSC on x86[_64]; it may go faster or // slower depending on CPU's turbo or power saving modes. Furthermore, to // read from CP15 on ARM a kernel modification or a driver is needed. // We can not require this from users of compiler-rt. // So on ARM we use clock_gettime(2) which gives the result in nanoseconds. // To get the measurements per second, we scale this by the number of // nanoseconds per second, pretending that the TSC frequency is 1GHz and // one TSC tick is 1 nanosecond. return NanosecondsPerSecond; } // The machine codes for some instructions used in runtime patching. enum class PatchOpcodes : uint32_t { PO_StpX0X30SP_m16e = 0xA9BF7BE0, // STP X0, X30, [SP, #-16]! PO_LdrW0_12 = 0x18000060, // LDR W0, #12 PO_LdrX16_12 = 0x58000070, // LDR X16, #12 PO_BlrX16 = 0xD63F0200, // BLR X16 PO_LdpX0X30SP_16 = 0xA8C17BE0, // LDP X0, X30, [SP], #16 PO_B32 = 0x14000008 // B #32 }; inline static bool patchSled(const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled, void (*TracingHook)()) XRAY_NEVER_INSTRUMENT { // When |Enable| == true, // We replace the following compile-time stub (sled): // // xray_sled_n: // B #32 // 7 NOPs (24 bytes) // // With the following runtime patch: // // xray_sled_n: // STP X0, X30, [SP, #-16]! ; PUSH {r0, lr} // LDR W0, #12 ; W0 := function ID // LDR X16,#12 ; X16 := address of the trampoline // BLR X16 // ;DATA: 32 bits of function ID // ;DATA: lower 32 bits of the address of the trampoline // ;DATA: higher 32 bits of the address of the trampoline // LDP X0, X30, [SP], #16 ; POP {r0, lr} // // Replacement of the first 4-byte instruction should be the last and atomic // operation, so that the user code which reaches the sled concurrently // either jumps over the whole sled, or executes the whole sled when the // latter is ready. // // When |Enable|==false, we set back the first instruction in the sled to be // B #32 uint32_t *FirstAddress = reinterpret_cast(Sled.Address); + uint32_t *CurAddress = FirstAddress + 1; if (Enable) { - uint32_t *CurAddress = FirstAddress + 1; *CurAddress = uint32_t(PatchOpcodes::PO_LdrW0_12); CurAddress++; *CurAddress = uint32_t(PatchOpcodes::PO_LdrX16_12); CurAddress++; *CurAddress = uint32_t(PatchOpcodes::PO_BlrX16); CurAddress++; *CurAddress = FuncId; CurAddress++; *reinterpret_cast(CurAddress) = TracingHook; CurAddress += 2; *CurAddress = uint32_t(PatchOpcodes::PO_LdpX0X30SP_16); + CurAddress++; std::atomic_store_explicit( reinterpret_cast *>(FirstAddress), uint32_t(PatchOpcodes::PO_StpX0X30SP_m16e), std::memory_order_release); } else { std::atomic_store_explicit( reinterpret_cast *>(FirstAddress), uint32_t(PatchOpcodes::PO_B32), std::memory_order_release); } + __clear_cache(reinterpret_cast(FirstAddress), + reinterpret_cast(CurAddress)); return true; } bool patchFunctionEntry(const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT { return patchSled(Enable, FuncId, Sled, __xray_FunctionEntry); } bool patchFunctionExit(const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT { return patchSled(Enable, FuncId, Sled, __xray_FunctionExit); } bool patchFunctionTailExit(const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT { // FIXME: In the future we'd need to distinguish between non-tail exits and // tail exits for better information preservation. return patchSled(Enable, FuncId, Sled, __xray_FunctionExit); } } // namespace __xray Index: projects/clang400-import/contrib/compiler-rt =================================================================== --- projects/clang400-import/contrib/compiler-rt (revision 312197) +++ projects/clang400-import/contrib/compiler-rt (revision 312198) Property changes on: projects/clang400-import/contrib/compiler-rt ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /vendor/compiler-rt/dist:r311836-312197