diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2021-07-29 20:15:26 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2021-07-29 20:15:26 +0000 |
commit | 344a3780b2e33f6ca763666c380202b18aab72a3 (patch) | |
tree | f0b203ee6eb71d7fdd792373e3c81eb18d6934dd /compiler-rt/lib/hwasan | |
parent | b60736ec1405bb0a8dd40989f67ef4c93da068ab (diff) | |
download | src-344a3780b2e33f6ca763666c380202b18aab72a3.tar.gz src-344a3780b2e33f6ca763666c380202b18aab72a3.zip |
Vendor import of llvm-project main 88e66fa60ae5, the last commit beforevendor/llvm-project/llvmorg-13-init-16847-g88e66fa60ae5vendor/llvm-project/llvmorg-12.0.1-rc2-0-ge7dac564cd0evendor/llvm-project/llvmorg-12.0.1-0-gfed41342a82f
the upstream release/13.x branch was created.
Diffstat (limited to 'compiler-rt/lib/hwasan')
26 files changed, 1010 insertions, 468 deletions
diff --git a/compiler-rt/lib/hwasan/hwasan.cpp b/compiler-rt/lib/hwasan/hwasan.cpp index c5322110cb66..cbe0dee66dcd 100644 --- a/compiler-rt/lib/hwasan/hwasan.cpp +++ b/compiler-rt/lib/hwasan/hwasan.cpp @@ -50,6 +50,11 @@ bool hwasan_init_is_running; int hwasan_report_count = 0; +uptr kLowShadowStart; +uptr kLowShadowEnd; +uptr kHighShadowStart; +uptr kHighShadowEnd; + void Flags::SetDefaults() { #define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; #include "hwasan_flags.inc" @@ -128,16 +133,11 @@ static void InitializeFlags() { if (common_flags()->help) parser.PrintFlagDescriptions(); } -static void HWAsanCheckFailed(const char *file, int line, const char *cond, - u64 v1, u64 v2) { - Report("HWAddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file, - line, cond, (uptr)v1, (uptr)v2); - PRINT_CURRENT_STACK_CHECK(); - Die(); +static void CheckUnwind() { + GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()); + stack.Print(); } -static constexpr uptr kMemoryUsageBufferSize = 4096; - static void HwasanFormatMemoryUsage(InternalScopedString &s) { HwasanThreadList &thread_list = hwasanThreadList(); auto thread_stats = thread_list.GetThreadStats(); @@ -155,6 +155,8 @@ static void HwasanFormatMemoryUsage(InternalScopedString &s) { } #if SANITIZER_ANDROID +static constexpr uptr kMemoryUsageBufferSize = 4096; + static char *memory_usage_buffer = nullptr; static void InitMemoryUsage() { @@ -171,7 +173,7 @@ void UpdateMemoryUsage() { return; if (!memory_usage_buffer) InitMemoryUsage(); - InternalScopedString s(kMemoryUsageBufferSize); + InternalScopedString s; HwasanFormatMemoryUsage(s); internal_strncpy(memory_usage_buffer, s.data(), kMemoryUsageBufferSize - 1); memory_usage_buffer[kMemoryUsageBufferSize - 1] = '\0'; @@ -180,6 +182,65 @@ void UpdateMemoryUsage() { void UpdateMemoryUsage() {} #endif +void HwasanAtExit() { + if (common_flags()->print_module_map) + DumpProcessMap(); + if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0)) + ReportStats(); + if (hwasan_report_count > 0) { + // ReportAtExitStatistics(); + if (common_flags()->exitcode) + internal__exit(common_flags()->exitcode); + } +} + +void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, void *uc, + uptr *registers_frame) { + InternalMmapVector<BufferedStackTrace> stack_buffer(1); + BufferedStackTrace *stack = stack_buffer.data(); + stack->Reset(); + stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal); + + // The second stack frame contains the failure __hwasan_check function, as + // we have a stack frame for the registers saved in __hwasan_tag_mismatch that + // we wish to ignore. This (currently) only occurs on AArch64, as x64 + // implementations use SIGTRAP to implement the failure, and thus do not go + // through the stack saver. + if (registers_frame && stack->trace && stack->size > 0) { + stack->trace++; + stack->size--; + } + + bool fatal = flags()->halt_on_error || !ai.recover; + ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal, + registers_frame); +} + +void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame, + size_t outsize) { + __hwasan::AccessInfo ai; + ai.is_store = access_info & 0x10; + ai.is_load = !ai.is_store; + ai.recover = access_info & 0x20; + ai.addr = addr; + if ((access_info & 0xf) == 0xf) + ai.size = outsize; + else + ai.size = 1 << (access_info & 0xf); + + HandleTagMismatch(ai, (uptr)__builtin_return_address(0), + (uptr)__builtin_frame_address(0), nullptr, registers_frame); + __builtin_unreachable(); +} + +Thread *GetCurrentThread() { + uptr *ThreadLongPtr = GetCurrentThreadLongPtr(); + if (UNLIKELY(*ThreadLongPtr == 0)) + return nullptr; + auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr; + return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next()); +} + } // namespace __hwasan using namespace __hwasan; @@ -219,7 +280,7 @@ static void InitLoadedGlobals() { static void InitInstrumentation() { if (hwasan_instrumentation_inited) return; - InitPrctl(); + InitializeOsSupport(); if (!InitShadow()) { Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n"); @@ -228,7 +289,6 @@ static void InitInstrumentation() { } InitThreads(); - hwasanThreadList().CreateCurrentThread(); hwasan_instrumentation_inited = 1; } @@ -271,7 +331,7 @@ void __hwasan_init() { InitializeFlags(); // Install tool-specific callbacks in sanitizer_common. - SetCheckFailedCallback(HWAsanCheckFailed); + SetCheckUnwindCallback(CheckUnwind); __sanitizer_set_report_path(common_flags()->log_path); @@ -493,12 +553,12 @@ extern "C" void *__hwasan_extra_spill_area() { } void __hwasan_print_memory_usage() { - InternalScopedString s(kMemoryUsageBufferSize); + InternalScopedString s; HwasanFormatMemoryUsage(s); Printf("%s\n", s.data()); } -static const u8 kFallbackTag = 0xBB; +static const u8 kFallbackTag = 0xBB & kTagMask; u8 __hwasan_generate_tag() { Thread *t = GetCurrentThread(); @@ -519,4 +579,12 @@ void __sanitizer_print_stack_trace() { GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()); stack.Print(); } + +// Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the +// rest of the mismatch handling code (C++). +void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame, + size_t outsize) { + __hwasan::HwasanTagMismatch(addr, access_info, registers_frame, outsize); +} + } // extern "C" diff --git a/compiler-rt/lib/hwasan/hwasan.h b/compiler-rt/lib/hwasan/hwasan.h index d4521efd089a..7338b696ad34 100644 --- a/compiler-rt/lib/hwasan/hwasan.h +++ b/compiler-rt/lib/hwasan/hwasan.h @@ -14,11 +14,12 @@ #ifndef HWASAN_H #define HWASAN_H +#include "hwasan_flags.h" +#include "hwasan_interface_internal.h" +#include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_stacktrace.h" -#include "hwasan_interface_internal.h" -#include "hwasan_flags.h" #include "ubsan/ubsan_platform.h" #ifndef HWASAN_CONTAINS_UBSAN @@ -35,10 +36,38 @@ typedef u8 tag_t; +#if defined(HWASAN_ALIASING_MODE) +# if !defined(__x86_64__) +# error Aliasing mode is only supported on x86_64 +# endif +// Tags are done in middle bits using userspace aliasing. +constexpr unsigned kAddressTagShift = 39; +constexpr unsigned kTagBits = 3; + +// The alias region is placed next to the shadow so the upper bits of all +// taggable addresses matches the upper bits of the shadow base. This shift +// value determines which upper bits must match. It has a floor of 44 since the +// shadow is always 8TB. +// TODO(morehouse): In alias mode we can shrink the shadow and use a +// simpler/faster shadow calculation. +constexpr unsigned kTaggableRegionCheckShift = + __sanitizer::Max(kAddressTagShift + kTagBits + 1U, 44U); +#elif defined(__x86_64__) +// Tags are done in upper bits using Intel LAM. +constexpr unsigned kAddressTagShift = 57; +constexpr unsigned kTagBits = 6; +#else // TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address // translation and can be used to store a tag. -const unsigned kAddressTagShift = 56; -const uptr kAddressTagMask = 0xFFUL << kAddressTagShift; +constexpr unsigned kAddressTagShift = 56; +constexpr unsigned kTagBits = 8; +#endif // defined(HWASAN_ALIASING_MODE) + +// Mask for extracting tag bits from the lower 8 bits. +constexpr uptr kTagMask = (1UL << kTagBits) - 1; + +// Mask for extracting tag bits from full pointers. +constexpr uptr kAddressTagMask = kTagMask << kAddressTagShift; // Minimal alignment of the shadow base address. Determines the space available // for threads and stack histories. This is an ABI constant. @@ -50,7 +79,7 @@ const unsigned kRecordFPLShift = 4; const unsigned kRecordFPModulus = 1 << (64 - kRecordFPShift + kRecordFPLShift); static inline tag_t GetTagFromPointer(uptr p) { - return p >> kAddressTagShift; + return (p >> kAddressTagShift) & kTagMask; } static inline uptr UntagAddr(uptr tagged_addr) { @@ -73,7 +102,7 @@ extern bool hwasan_init_is_running; extern int hwasan_report_count; bool InitShadow(); -void InitPrctl(); +void InitializeOsSupport(); void InitThreads(); void InitializeInterceptors(); @@ -105,17 +134,9 @@ void InstallAtExitHandler(); if (hwasan_inited) \ stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal) -#define GET_FATAL_STACK_TRACE_HERE \ - GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()) - -#define PRINT_CURRENT_STACK_CHECK() \ - { \ - GET_FATAL_STACK_TRACE_HERE; \ - stack.Print(); \ - } - void HwasanTSDInit(); void HwasanTSDThreadInit(); +void HwasanAtExit(); void HwasanOnDeadlySignal(int signo, void *info, void *context); @@ -125,6 +146,26 @@ void AppendToErrorMessageBuffer(const char *buffer); void AndroidTestTlsSlot(); +// This is a compiler-generated struct that can be shared between hwasan +// implementations. +struct AccessInfo { + uptr addr; + uptr size; + bool is_store; + bool is_load; + bool recover; +}; + +// Given access info and frame information, unwind the stack and report the tag +// mismatch. +void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, void *uc, + uptr *registers_frame = nullptr); + +// This dispatches to HandleTagMismatch but sets up the AccessInfo, program +// counter, and frame pointer. +void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame, + size_t outsize); + } // namespace __hwasan #define HWASAN_MALLOC_HOOK(ptr, size) \ @@ -162,4 +203,12 @@ typedef struct __hw_jmp_buf_struct __hw_jmp_buf[1]; typedef struct __hw_jmp_buf_struct __hw_sigjmp_buf[1]; #endif // HWASAN_WITH_INTERCEPTORS && __aarch64__ +#define ENSURE_HWASAN_INITED() \ + do { \ + CHECK(!hwasan_init_is_running); \ + if (!hwasan_inited) { \ + __hwasan_init(); \ + } \ + } while (0) + #endif // HWASAN_H diff --git a/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp b/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp new file mode 100644 index 000000000000..6c2a6077866f --- /dev/null +++ b/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp @@ -0,0 +1,172 @@ +//===-- hwasan_allocation_functions.cpp -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Definitions for __sanitizer allocation functions. +// +//===----------------------------------------------------------------------===// + +#include "hwasan.h" +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_tls_get_addr.h" + +using namespace __hwasan; + +static uptr allocated_for_dlsym; +static const uptr kDlsymAllocPoolSize = 1024; +static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; + +static bool IsInDlsymAllocPool(const void *ptr) { + uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; + return off < sizeof(alloc_memory_for_dlsym); +} + +static void *AllocateFromLocalPool(uptr size_in_bytes) { + uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; + void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; + allocated_for_dlsym += size_in_words; + CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); + return mem; +} + +int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) { + GET_MALLOC_STACK_TRACE; + CHECK_NE(memptr, 0); + int res = hwasan_posix_memalign(memptr, alignment, size, &stack); + return res; +} + +void *__sanitizer_memalign(uptr alignment, uptr size) { + GET_MALLOC_STACK_TRACE; + return hwasan_memalign(alignment, size, &stack); +} + +void *__sanitizer_aligned_alloc(uptr alignment, uptr size) { + GET_MALLOC_STACK_TRACE; + return hwasan_aligned_alloc(alignment, size, &stack); +} + +void *__sanitizer___libc_memalign(uptr alignment, uptr size) { + GET_MALLOC_STACK_TRACE; + void *ptr = hwasan_memalign(alignment, size, &stack); + if (ptr) + DTLS_on_libc_memalign(ptr, size); + return ptr; +} + +void *__sanitizer_valloc(uptr size) { + GET_MALLOC_STACK_TRACE; + return hwasan_valloc(size, &stack); +} + +void *__sanitizer_pvalloc(uptr size) { + GET_MALLOC_STACK_TRACE; + return hwasan_pvalloc(size, &stack); +} + +void __sanitizer_free(void *ptr) { + GET_MALLOC_STACK_TRACE; + if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) + return; + hwasan_free(ptr, &stack); +} + +void __sanitizer_cfree(void *ptr) { + GET_MALLOC_STACK_TRACE; + if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) + return; + hwasan_free(ptr, &stack); +} + +uptr __sanitizer_malloc_usable_size(const void *ptr) { + return __sanitizer_get_allocated_size(ptr); +} + +struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() { + __sanitizer_struct_mallinfo sret; + internal_memset(&sret, 0, sizeof(sret)); + return sret; +} + +int __sanitizer_mallopt(int cmd, int value) { return 0; } + +void __sanitizer_malloc_stats(void) { + // FIXME: implement, but don't call REAL(malloc_stats)! +} + +void *__sanitizer_calloc(uptr nmemb, uptr size) { + GET_MALLOC_STACK_TRACE; + if (UNLIKELY(!hwasan_inited)) + // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. + return AllocateFromLocalPool(nmemb * size); + return hwasan_calloc(nmemb, size, &stack); +} + +void *__sanitizer_realloc(void *ptr, uptr size) { + GET_MALLOC_STACK_TRACE; + if (UNLIKELY(IsInDlsymAllocPool(ptr))) { + uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; + uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); + void *new_ptr; + if (UNLIKELY(!hwasan_inited)) { + new_ptr = AllocateFromLocalPool(copy_size); + } else { + copy_size = size; + new_ptr = hwasan_malloc(copy_size, &stack); + } + internal_memcpy(new_ptr, ptr, copy_size); + return new_ptr; + } + return hwasan_realloc(ptr, size, &stack); +} + +void *__sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) { + GET_MALLOC_STACK_TRACE; + return hwasan_reallocarray(ptr, nmemb, size, &stack); +} + +void *__sanitizer_malloc(uptr size) { + GET_MALLOC_STACK_TRACE; + if (UNLIKELY(!hwasan_init_is_running)) + ENSURE_HWASAN_INITED(); + if (UNLIKELY(!hwasan_inited)) + // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. + return AllocateFromLocalPool(size); + return hwasan_malloc(size, &stack); +} + +#if HWASAN_WITH_INTERCEPTORS +# define INTERCEPTOR_ALIAS(RET, FN, ARGS...) \ + extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS) \ + ALIAS("__sanitizer_" #FN); \ + extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN( \ + ARGS) ALIAS("__sanitizer_" #FN) + +INTERCEPTOR_ALIAS(int, posix_memalign, void **memptr, SIZE_T alignment, + SIZE_T size); +INTERCEPTOR_ALIAS(void *, aligned_alloc, SIZE_T alignment, SIZE_T size); +INTERCEPTOR_ALIAS(void *, __libc_memalign, SIZE_T alignment, SIZE_T size); +INTERCEPTOR_ALIAS(void *, valloc, SIZE_T size); +INTERCEPTOR_ALIAS(void, free, void *ptr); +INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr); +INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size); +INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size); +INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size); +INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size); + +# if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR_ALIAS(void *, memalign, SIZE_T alignment, SIZE_T size); +INTERCEPTOR_ALIAS(void *, pvalloc, SIZE_T size); +INTERCEPTOR_ALIAS(void, cfree, void *ptr); +INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo); +INTERCEPTOR_ALIAS(int, mallopt, int cmd, int value); +INTERCEPTOR_ALIAS(void, malloc_stats, void); +# endif +#endif // #if HWASAN_WITH_INTERCEPTORS diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.cpp b/compiler-rt/lib/hwasan/hwasan_allocator.cpp index 0b6b7347892e..ef6d4d6c7678 100644 --- a/compiler-rt/lib/hwasan/hwasan_allocator.cpp +++ b/compiler-rt/lib/hwasan/hwasan_allocator.cpp @@ -29,8 +29,8 @@ static AllocatorCache fallback_allocator_cache; static SpinMutex fallback_mutex; static atomic_uint8_t hwasan_allocator_tagging_enabled; -static const tag_t kFallbackAllocTag = 0xBB; -static const tag_t kFallbackFreeTag = 0xBC; +static constexpr tag_t kFallbackAllocTag = 0xBB & kTagMask; +static constexpr tag_t kFallbackFreeTag = 0xBC; enum RightAlignMode { kRightAlignNever, @@ -80,11 +80,29 @@ void GetAllocatorStats(AllocatorStatCounters s) { allocator.GetStats(s); } +uptr GetAliasRegionStart() { +#if defined(HWASAN_ALIASING_MODE) + constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1); + uptr AliasRegionStart = + __hwasan_shadow_memory_dynamic_address + kAliasRegionOffset; + + CHECK_EQ(AliasRegionStart >> kTaggableRegionCheckShift, + __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift); + CHECK_EQ( + (AliasRegionStart + kAliasRegionOffset - 1) >> kTaggableRegionCheckShift, + __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift); + return AliasRegionStart; +#else + return 0; +#endif +} + void HwasanAllocatorInit() { atomic_store_relaxed(&hwasan_allocator_tagging_enabled, !flags()->disable_allocator_tagging); SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); - allocator.Init(common_flags()->allocator_release_to_os_interval_ms); + allocator.Init(common_flags()->allocator_release_to_os_interval_ms, + GetAliasRegionStart()); for (uptr i = 0; i < sizeof(tail_magic); i++) tail_magic[i] = GetCurrentThread()->GenerateRandomTag(); } @@ -148,7 +166,8 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, // Tagging can only be skipped when both tag_in_malloc and tag_in_free are // false. When tag_in_malloc = false and tag_in_free = true malloc needs to // retag to 0. - if ((flags()->tag_in_malloc || flags()->tag_in_free) && + if (InTaggableRegion(reinterpret_cast<uptr>(user_ptr)) && + (flags()->tag_in_malloc || flags()->tag_in_free) && atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) { if (flags()->tag_in_malloc && malloc_bisect(stack, orig_size)) { tag_t tag = t ? t->GenerateRandomTag() : kFallbackAllocTag; @@ -175,6 +194,8 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, static bool PointerAndMemoryTagsMatch(void *tagged_ptr) { CHECK(tagged_ptr); uptr tagged_uptr = reinterpret_cast<uptr>(tagged_ptr); + if (!InTaggableRegion(tagged_uptr)) + return true; tag_t mem_tag = *reinterpret_cast<tag_t *>( MemToShadow(reinterpret_cast<uptr>(UntagPtr(tagged_ptr)))); return PossiblyShortTagMatches(mem_tag, tagged_uptr, 1); @@ -187,9 +208,12 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { if (!PointerAndMemoryTagsMatch(tagged_ptr)) ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr)); - void *untagged_ptr = UntagPtr(tagged_ptr); + void *untagged_ptr = InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr)) + ? UntagPtr(tagged_ptr) + : tagged_ptr; void *aligned_ptr = reinterpret_cast<void *>( RoundDownTo(reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment)); + tag_t pointer_tag = GetTagFromPointer(reinterpret_cast<uptr>(tagged_ptr)); Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(aligned_ptr)); uptr orig_size = meta->get_requested_size(); @@ -219,10 +243,27 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { Min(TaggedSize(orig_size), (uptr)flags()->max_free_fill_size); internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size); } - if (flags()->tag_in_free && malloc_bisect(stack, 0) && - atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) + if (InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr)) && + flags()->tag_in_free && malloc_bisect(stack, 0) && + atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) { + // Always store full 8-bit tags on free to maximize UAF detection. + tag_t tag; + if (t) { + // Make sure we are not using a short granule tag as a poison tag. This + // would make us attempt to read the memory on a UaF. + // The tag can be zero if tagging is disabled on this thread. + do { + tag = t->GenerateRandomTag(/*num_bits=*/8); + } while ( + UNLIKELY((tag < kShadowAlignment || tag == pointer_tag) && tag != 0)); + } else { + static_assert(kFallbackFreeTag >= kShadowAlignment, + "fallback tag must not be a short granule tag."); + tag = kFallbackFreeTag; + } TagMemoryAligned(reinterpret_cast<uptr>(aligned_ptr), TaggedSize(orig_size), - t ? t->GenerateRandomTag() : kFallbackFreeTag); + tag); + } if (t) { allocator.Deallocate(t->allocator_cache(), aligned_ptr); if (auto *ha = t->heap_allocations()) @@ -365,7 +406,7 @@ int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, // OOM error is already taken care of by HwasanAllocate. return errno_ENOMEM; CHECK(IsAligned((uptr)ptr, alignment)); - *(void **)UntagPtr(memptr) = ptr; + *memptr = ptr; return 0; } diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.h b/compiler-rt/lib/hwasan/hwasan_allocator.h index 43670a6a3fb7..35c3d6b4bf43 100644 --- a/compiler-rt/lib/hwasan/hwasan_allocator.h +++ b/compiler-rt/lib/hwasan/hwasan_allocator.h @@ -13,13 +13,16 @@ #ifndef HWASAN_ALLOCATOR_H #define HWASAN_ALLOCATOR_H +#include "hwasan.h" +#include "hwasan_interface_internal.h" +#include "hwasan_mapping.h" +#include "hwasan_poisoning.h" #include "sanitizer_common/sanitizer_allocator.h" #include "sanitizer_common/sanitizer_allocator_checks.h" #include "sanitizer_common/sanitizer_allocator_interface.h" #include "sanitizer_common/sanitizer_allocator_report.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_ring_buffer.h" -#include "hwasan_poisoning.h" #if !defined(__aarch64__) && !defined(__x86_64__) #error Unsupported platform @@ -55,7 +58,12 @@ static const uptr kMaxAllowedMallocSize = 1UL << 40; // 1T struct AP64 { static const uptr kSpaceBeg = ~0ULL; + +#if defined(HWASAN_ALIASING_MODE) + static const uptr kSpaceSize = 1ULL << kAddressTagShift; +#else static const uptr kSpaceSize = 0x2000000000ULL; +#endif static const uptr kMetadataSize = sizeof(Metadata); typedef __sanitizer::VeryDenseSizeClassMap SizeClassMap; using AddressSpaceView = LocalAddressSpaceView; @@ -102,6 +110,16 @@ typedef RingBuffer<HeapAllocationRecord> HeapAllocationsRingBuffer; void GetAllocatorStats(AllocatorStatCounters s); +inline bool InTaggableRegion(uptr addr) { +#if defined(HWASAN_ALIASING_MODE) + // Aliases are mapped next to shadow so that the upper bits match the shadow + // base. + return (addr >> kTaggableRegionCheckShift) == + (GetShadowOffset() >> kTaggableRegionCheckShift); +#endif + return true; +} + } // namespace __hwasan #endif // HWASAN_ALLOCATOR_H diff --git a/compiler-rt/lib/hwasan/hwasan_blacklist.txt b/compiler-rt/lib/hwasan/hwasan_blacklist.txt deleted file mode 100644 index 395ba28f0212..000000000000 --- a/compiler-rt/lib/hwasan/hwasan_blacklist.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Blacklist for HWAddressSanitizer. Turns off instrumentation of particular -# functions or sources. Use with care. You may set location of blacklist -# at compile-time using -fsanitize-blacklist=<path> flag. - -# Example usage: -# fun:*bad_function_name* -# src:file_with_tricky_code.cc diff --git a/compiler-rt/lib/hwasan/hwasan_checks.h b/compiler-rt/lib/hwasan/hwasan_checks.h index a8de0fef20f0..ab543ea88beb 100644 --- a/compiler-rt/lib/hwasan/hwasan_checks.h +++ b/compiler-rt/lib/hwasan/hwasan_checks.h @@ -13,6 +13,7 @@ #ifndef HWASAN_CHECKS_H #define HWASAN_CHECKS_H +#include "hwasan_allocator.h" #include "hwasan_mapping.h" #include "sanitizer_common/sanitizer_common.h" @@ -81,6 +82,8 @@ enum class AccessType { Load, Store }; template <ErrorAction EA, AccessType AT, unsigned LogSize> __attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) { + if (!InTaggableRegion(p)) + return; uptr ptr_raw = p & ~kAddressTagMask; tag_t mem_tag = *(tag_t *)MemToShadow(ptr_raw); if (UNLIKELY(!PossiblyShortTagMatches(mem_tag, p, 1 << LogSize))) { @@ -94,7 +97,7 @@ __attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) { template <ErrorAction EA, AccessType AT> __attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p, uptr sz) { - if (sz == 0) + if (sz == 0 || !InTaggableRegion(p)) return; tag_t ptr_tag = GetTagFromPointer(p); uptr ptr_raw = p & ~kAddressTagMask; diff --git a/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp b/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp index 12730b29bae3..7642ba6c0bf0 100644 --- a/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp +++ b/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp @@ -12,15 +12,17 @@ /// //===----------------------------------------------------------------------===// -#include "hwasan.h" #include "hwasan_dynamic_shadow.h" -#include "hwasan_mapping.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_posix.h" #include <elf.h> #include <link.h> +#include "hwasan.h" +#include "hwasan_mapping.h" +#include "hwasan_thread_list.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_posix.h" + // The code in this file needs to run in an unrelocated binary. It should not // access any external symbol, including its own non-hidden globals. @@ -111,12 +113,27 @@ uptr FindDynamicShadowStart(uptr shadow_size_bytes) { } } // namespace __hwasan + +#elif SANITIZER_FUCHSIA + +namespace __hwasan { + +void InitShadowGOT() {} + +} // namespace __hwasan + #else namespace __hwasan { void InitShadowGOT() {} uptr FindDynamicShadowStart(uptr shadow_size_bytes) { +# if defined(HWASAN_ALIASING_MODE) + constexpr uptr kAliasSize = 1ULL << kAddressTagShift; + constexpr uptr kNumAliases = 1ULL << kTagBits; + return MapDynamicShadowAndAliases(shadow_size_bytes, kAliasSize, kNumAliases, + RingBufferSize()); +# endif return MapDynamicShadow(shadow_size_bytes, kShadowScale, kShadowBaseAlignment, kHighMemEnd); } diff --git a/compiler-rt/lib/hwasan/hwasan_flags.h b/compiler-rt/lib/hwasan/hwasan_flags.h index 0a6998f675d6..b17750158d02 100644 --- a/compiler-rt/lib/hwasan/hwasan_flags.h +++ b/compiler-rt/lib/hwasan/hwasan_flags.h @@ -12,6 +12,8 @@ #ifndef HWASAN_FLAGS_H #define HWASAN_FLAGS_H +#include "sanitizer_common/sanitizer_internal_defs.h" + namespace __hwasan { struct Flags { diff --git a/compiler-rt/lib/hwasan/hwasan_flags.inc b/compiler-rt/lib/hwasan/hwasan_flags.inc index 8e431d9c4ff9..18ea47f981be 100644 --- a/compiler-rt/lib/hwasan/hwasan_flags.inc +++ b/compiler-rt/lib/hwasan/hwasan_flags.inc @@ -72,3 +72,12 @@ HWASAN_FLAG(uptr, malloc_bisect_right, 0, HWASAN_FLAG(bool, malloc_bisect_dump, false, "Print all allocations within [malloc_bisect_left, " "malloc_bisect_right] range ") + + +// Exit if we fail to enable the AArch64 kernel ABI relaxation which allows +// tagged pointers in syscalls. This is the default, but being able to disable +// that behaviour is useful for running the testsuite on more platforms (the +// testsuite can run since we manually ensure any pointer arguments to syscalls +// are untagged before the call. +HWASAN_FLAG(bool, fail_without_syscall_abi, true, + "Exit if fail to request relaxed syscall ABI.") diff --git a/compiler-rt/lib/hwasan/hwasan_fuchsia.cpp b/compiler-rt/lib/hwasan/hwasan_fuchsia.cpp new file mode 100644 index 000000000000..e299a7e862eb --- /dev/null +++ b/compiler-rt/lib/hwasan/hwasan_fuchsia.cpp @@ -0,0 +1,213 @@ +//===-- hwasan_fuchsia.cpp --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file is a part of HWAddressSanitizer and contains Fuchsia-specific +/// code. +/// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_fuchsia.h" +#if SANITIZER_FUCHSIA + +#include "hwasan.h" +#include "hwasan_interface_internal.h" +#include "hwasan_report.h" +#include "hwasan_thread.h" +#include "hwasan_thread_list.h" + +// This TLS variable contains the location of the stack ring buffer and can be +// used to always find the hwasan thread object associated with the current +// running thread. +[[gnu::tls_model("initial-exec")]] +SANITIZER_INTERFACE_ATTRIBUTE +THREADLOCAL uptr __hwasan_tls; + +namespace __hwasan { + +bool InitShadow() { + __sanitizer::InitShadowBounds(); + CHECK_NE(__sanitizer::ShadowBounds.shadow_limit, 0); + + // These variables are used by MemIsShadow for asserting we have a correct + // shadow address. On Fuchsia, we only have one region of shadow, so the + // bounds of Low shadow can be zero while High shadow represents the true + // bounds. Note that these are inclusive ranges. + kLowShadowStart = 0; + kLowShadowEnd = 0; + kHighShadowStart = __sanitizer::ShadowBounds.shadow_base; + kHighShadowEnd = __sanitizer::ShadowBounds.shadow_limit - 1; + + return true; +} + +bool MemIsApp(uptr p) { + CHECK(GetTagFromPointer(p) == 0); + return __sanitizer::ShadowBounds.shadow_limit <= p && + p <= (__sanitizer::ShadowBounds.memory_limit - 1); +} + +// These are known parameters passed to the hwasan runtime on thread creation. +struct Thread::InitState { + uptr stack_bottom, stack_top; +}; + +static void FinishThreadInitialization(Thread *thread); + +void InitThreads() { + // This is the minimal alignment needed for the storage where hwasan threads + // and their stack ring buffers are placed. This alignment is necessary so the + // stack ring buffer can perform a simple calculation to get the next element + // in the RB. The instructions for this calculation are emitted by the + // compiler. (Full explanation in hwasan_thread_list.h.) + uptr alloc_size = UINT64_C(1) << kShadowBaseAlignment; + uptr thread_start = reinterpret_cast<uptr>( + MmapAlignedOrDieOnFatalError(alloc_size, alloc_size, __func__)); + + InitThreadList(thread_start, alloc_size); + + // Create the hwasan thread object for the current (main) thread. Stack info + // for this thread is known from information passed via + // __sanitizer_startup_hook. + const Thread::InitState state = { + .stack_bottom = __sanitizer::MainThreadStackBase, + .stack_top = + __sanitizer::MainThreadStackBase + __sanitizer::MainThreadStackSize, + }; + FinishThreadInitialization(hwasanThreadList().CreateCurrentThread(&state)); +} + +uptr *GetCurrentThreadLongPtr() { return &__hwasan_tls; } + +// This is called from the parent thread before the new thread is created. Here +// we can propagate known info like the stack bounds to Thread::Init before +// jumping into the thread. We cannot initialize the stack ring buffer yet since +// we have not entered the new thread. +static void *BeforeThreadCreateHook(uptr user_id, bool detached, + const char *name, uptr stack_bottom, + uptr stack_size) { + const Thread::InitState state = { + .stack_bottom = stack_bottom, + .stack_top = stack_bottom + stack_size, + }; + return hwasanThreadList().CreateCurrentThread(&state); +} + +// This sets the stack top and bottom according to the InitState passed to +// CreateCurrentThread above. +void Thread::InitStackAndTls(const InitState *state) { + CHECK_NE(state->stack_bottom, 0); + CHECK_NE(state->stack_top, 0); + stack_bottom_ = state->stack_bottom; + stack_top_ = state->stack_top; + tls_end_ = tls_begin_ = 0; +} + +// This is called after creating a new thread with the pointer returned by +// BeforeThreadCreateHook. We are still in the creating thread and should check +// if it was actually created correctly. +static void ThreadCreateHook(void *hook, bool aborted) { + Thread *thread = static_cast<Thread *>(hook); + if (!aborted) { + // The thread was created successfully. + // ThreadStartHook can already be running in the new thread. + } else { + // The thread wasn't created after all. + // Clean up everything we set up in BeforeThreadCreateHook. + atomic_signal_fence(memory_order_seq_cst); + hwasanThreadList().ReleaseThread(thread); + } +} + +// This is called in the newly-created thread before it runs anything else, +// with the pointer returned by BeforeThreadCreateHook (above). Here we can +// setup the stack ring buffer. +static void ThreadStartHook(void *hook, thrd_t self) { + Thread *thread = static_cast<Thread *>(hook); + FinishThreadInitialization(thread); + thread->InitRandomState(); +} + +// This is the function that sets up the stack ring buffer and enables us to use +// GetCurrentThread. This function should only be called while IN the thread +// that we want to create the hwasan thread object for so __hwasan_tls can be +// properly referenced. +static void FinishThreadInitialization(Thread *thread) { + CHECK_NE(thread, nullptr); + + // The ring buffer is located immediately before the thread object. + uptr stack_buffer_size = hwasanThreadList().GetRingBufferSize(); + uptr stack_buffer_start = reinterpret_cast<uptr>(thread) - stack_buffer_size; + thread->InitStackRingBuffer(stack_buffer_start, stack_buffer_size); +} + +static void ThreadExitHook(void *hook, thrd_t self) { + Thread *thread = static_cast<Thread *>(hook); + atomic_signal_fence(memory_order_seq_cst); + hwasanThreadList().ReleaseThread(thread); +} + +uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { + CHECK(IsAligned(p, kShadowAlignment)); + CHECK(IsAligned(size, kShadowAlignment)); + __sanitizer_fill_shadow(p, size, tag, + common_flags()->clear_shadow_mmap_threshold); + return AddTagToPointer(p, tag); +} + +// Not implemented because Fuchsia does not use signal handlers. +void HwasanOnDeadlySignal(int signo, void *info, void *context) {} + +// Not implemented because Fuchsia does not use interceptors. +void InitializeInterceptors() {} + +// Not implemented because this is only relevant for Android. +void AndroidTestTlsSlot() {} + +// TSD was normally used on linux as a means of calling the hwasan thread exit +// handler passed to pthread_key_create. This is not needed on Fuchsia because +// we will be using __sanitizer_thread_exit_hook. +void HwasanTSDInit() {} +void HwasanTSDThreadInit() {} + +// On linux, this just would call `atexit(HwasanAtExit)`. The functions in +// HwasanAtExit are unimplemented for Fuchsia and effectively no-ops, so this +// function is unneeded. +void InstallAtExitHandler() {} + +// TODO(fxbug.dev/81499): Once we finalize the tagged pointer ABI in zircon, we should come back +// here and implement the appropriate check that TBI is enabled. +void InitializeOsSupport() {} + +} // namespace __hwasan + +extern "C" { + +void *__sanitizer_before_thread_create_hook(thrd_t thread, bool detached, + const char *name, void *stack_base, + size_t stack_size) { + return __hwasan::BeforeThreadCreateHook( + reinterpret_cast<uptr>(thread), detached, name, + reinterpret_cast<uptr>(stack_base), stack_size); +} + +void __sanitizer_thread_create_hook(void *hook, thrd_t thread, int error) { + __hwasan::ThreadCreateHook(hook, error != thrd_success); +} + +void __sanitizer_thread_start_hook(void *hook, thrd_t self) { + __hwasan::ThreadStartHook(hook, reinterpret_cast<uptr>(self)); +} + +void __sanitizer_thread_exit_hook(void *hook, thrd_t self) { + __hwasan::ThreadExitHook(hook, self); +} + +} // extern "C" + +#endif // SANITIZER_FUCHSIA diff --git a/compiler-rt/lib/hwasan/hwasan_ignorelist.txt b/compiler-rt/lib/hwasan/hwasan_ignorelist.txt new file mode 100644 index 000000000000..70590c970f55 --- /dev/null +++ b/compiler-rt/lib/hwasan/hwasan_ignorelist.txt @@ -0,0 +1,7 @@ +# Ignorelist for HWAddressSanitizer. Turns off instrumentation of particular +# functions or sources. Use with care. You may set location of ignorelist +# at compile-time using -fsanitize-ignorelist=<path> flag. + +# Example usage: +# fun:*bad_function_name* +# src:file_with_tricky_code.cc diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp index 44e569ee6d72..68f8adec0776 100644 --- a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp +++ b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp @@ -16,192 +16,14 @@ #include "interception/interception.h" #include "hwasan.h" -#include "hwasan_allocator.h" -#include "hwasan_mapping.h" #include "hwasan_thread.h" -#include "hwasan_poisoning.h" -#include "hwasan_report.h" -#include "sanitizer_common/sanitizer_platform_limits_posix.h" -#include "sanitizer_common/sanitizer_allocator.h" -#include "sanitizer_common/sanitizer_allocator_interface.h" -#include "sanitizer_common/sanitizer_allocator_internal.h" -#include "sanitizer_common/sanitizer_atomic.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_stackdepot.h" -#include "sanitizer_common/sanitizer_libc.h" -#include "sanitizer_common/sanitizer_linux.h" -#include "sanitizer_common/sanitizer_tls_get_addr.h" -#include <stdarg.h> -// ACHTUNG! No other system header includes in this file. -// Ideally, we should get rid of stdarg.h as well. +#if !SANITIZER_FUCHSIA using namespace __hwasan; -using __sanitizer::memory_order; -using __sanitizer::atomic_load; -using __sanitizer::atomic_store; -using __sanitizer::atomic_uintptr_t; - -static uptr allocated_for_dlsym; -static const uptr kDlsymAllocPoolSize = 1024; -static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; - -static bool IsInDlsymAllocPool(const void *ptr) { - uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - return off < sizeof(alloc_memory_for_dlsym); -} - -static void *AllocateFromLocalPool(uptr size_in_bytes) { - uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; - void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; - allocated_for_dlsym += size_in_words; - CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); - return mem; -} - -#define ENSURE_HWASAN_INITED() do { \ - CHECK(!hwasan_init_is_running); \ - if (!hwasan_inited) { \ - __hwasan_init(); \ - } \ -} while (0) - - -int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) { - GET_MALLOC_STACK_TRACE; - CHECK_NE(memptr, 0); - int res = hwasan_posix_memalign(memptr, alignment, size, &stack); - return res; -} - -void * __sanitizer_memalign(uptr alignment, uptr size) { - GET_MALLOC_STACK_TRACE; - return hwasan_memalign(alignment, size, &stack); -} - -void * __sanitizer_aligned_alloc(uptr alignment, uptr size) { - GET_MALLOC_STACK_TRACE; - return hwasan_aligned_alloc(alignment, size, &stack); -} - -void * __sanitizer___libc_memalign(uptr alignment, uptr size) { - GET_MALLOC_STACK_TRACE; - void *ptr = hwasan_memalign(alignment, size, &stack); - if (ptr) - DTLS_on_libc_memalign(ptr, size); - return ptr; -} - -void * __sanitizer_valloc(uptr size) { - GET_MALLOC_STACK_TRACE; - return hwasan_valloc(size, &stack); -} - -void * __sanitizer_pvalloc(uptr size) { - GET_MALLOC_STACK_TRACE; - return hwasan_pvalloc(size, &stack); -} - -void __sanitizer_free(void *ptr) { - GET_MALLOC_STACK_TRACE; - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; - hwasan_free(ptr, &stack); -} - -void __sanitizer_cfree(void *ptr) { - GET_MALLOC_STACK_TRACE; - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; - hwasan_free(ptr, &stack); -} - -uptr __sanitizer_malloc_usable_size(const void *ptr) { - return __sanitizer_get_allocated_size(ptr); -} - -struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() { - __sanitizer_struct_mallinfo sret; - internal_memset(&sret, 0, sizeof(sret)); - return sret; -} - -int __sanitizer_mallopt(int cmd, int value) { - return 0; -} - -void __sanitizer_malloc_stats(void) { - // FIXME: implement, but don't call REAL(malloc_stats)! -} - -void * __sanitizer_calloc(uptr nmemb, uptr size) { - GET_MALLOC_STACK_TRACE; - if (UNLIKELY(!hwasan_inited)) - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - return AllocateFromLocalPool(nmemb * size); - return hwasan_calloc(nmemb, size, &stack); -} - -void * __sanitizer_realloc(void *ptr, uptr size) { - GET_MALLOC_STACK_TRACE; - if (UNLIKELY(IsInDlsymAllocPool(ptr))) { - uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); - void *new_ptr; - if (UNLIKELY(!hwasan_inited)) { - new_ptr = AllocateFromLocalPool(copy_size); - } else { - copy_size = size; - new_ptr = hwasan_malloc(copy_size, &stack); - } - internal_memcpy(new_ptr, ptr, copy_size); - return new_ptr; - } - return hwasan_realloc(ptr, size, &stack); -} - -void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) { - GET_MALLOC_STACK_TRACE; - return hwasan_reallocarray(ptr, nmemb, size, &stack); -} - -void * __sanitizer_malloc(uptr size) { - GET_MALLOC_STACK_TRACE; - if (UNLIKELY(!hwasan_init_is_running)) - ENSURE_HWASAN_INITED(); - if (UNLIKELY(!hwasan_inited)) - // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. - return AllocateFromLocalPool(size); - return hwasan_malloc(size, &stack); -} - #if HWASAN_WITH_INTERCEPTORS -#define INTERCEPTOR_ALIAS(RET, FN, ARGS...) \ - extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS) \ - ALIAS("__sanitizer_" #FN); \ - extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN( \ - ARGS) ALIAS("__sanitizer_" #FN) - -INTERCEPTOR_ALIAS(int, posix_memalign, void **memptr, SIZE_T alignment, - SIZE_T size); -INTERCEPTOR_ALIAS(void *, aligned_alloc, SIZE_T alignment, SIZE_T size); -INTERCEPTOR_ALIAS(void *, __libc_memalign, SIZE_T alignment, SIZE_T size); -INTERCEPTOR_ALIAS(void *, valloc, SIZE_T size); -INTERCEPTOR_ALIAS(void, free, void *ptr); -INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr); -INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size); -INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size); -INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size); -INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size); - -#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD -INTERCEPTOR_ALIAS(void *, memalign, SIZE_T alignment, SIZE_T size); -INTERCEPTOR_ALIAS(void *, pvalloc, SIZE_T size); -INTERCEPTOR_ALIAS(void, cfree, void *ptr); -INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo); -INTERCEPTOR_ALIAS(int, mallopt, int cmd, int value); -INTERCEPTOR_ALIAS(void, malloc_stats, void); -#endif struct ThreadStartArg { thread_callback_t callback; @@ -221,8 +43,7 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), ThreadStartArg *A = reinterpret_cast<ThreadStartArg *> (MmapOrDie( GetPageSizeCached(), "pthread_create")); *A = {callback, param}; - int res = REAL(pthread_create)(UntagPtr(th), UntagPtr(attr), - &HwasanThreadStartFunc, A); + int res = REAL(pthread_create)(th, attr, &HwasanThreadStartFunc, A); return res; } @@ -347,3 +168,5 @@ void InitializeInterceptors() { inited = 1; } } // namespace __hwasan + +#endif // #if !SANITIZER_FUCHSIA diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S b/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S index 23d565936d87..fd20825e3dac 100644 --- a/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S +++ b/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S @@ -1,4 +1,5 @@ #include "sanitizer_common/sanitizer_asm.h" +#include "builtins/assembly.h" #if defined(__linux__) && HWASAN_WITH_INTERCEPTORS #define COMMON_INTERCEPTOR_SPILL_AREA __hwasan_extra_spill_area @@ -9,3 +10,5 @@ #endif NO_EXEC_STACK_DIRECTIVE + +GNU_PROPERTY_BTI_PAC diff --git a/compiler-rt/lib/hwasan/hwasan_linux.cpp b/compiler-rt/lib/hwasan/hwasan_linux.cpp index e99926d355cf..e22723529f44 100644 --- a/compiler-rt/lib/hwasan/hwasan_linux.cpp +++ b/compiler-rt/lib/hwasan/hwasan_linux.cpp @@ -69,10 +69,6 @@ static void ProtectGap(uptr addr, uptr size) { uptr kLowMemStart; uptr kLowMemEnd; -uptr kLowShadowEnd; -uptr kLowShadowStart; -uptr kHighShadowStart; -uptr kHighShadowEnd; uptr kHighMemStart; uptr kHighMemEnd; @@ -114,37 +110,59 @@ static void InitializeShadowBaseAddress(uptr shadow_size_bytes) { FindDynamicShadowStart(shadow_size_bytes); } -void InitPrctl() { +void InitializeOsSupport() { #define PR_SET_TAGGED_ADDR_CTRL 55 #define PR_GET_TAGGED_ADDR_CTRL 56 #define PR_TAGGED_ADDR_ENABLE (1UL << 0) // Check we're running on a kernel that can use the tagged address ABI. - if (internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0) == (uptr)-1 && - errno == EINVAL) { -#if SANITIZER_ANDROID + int local_errno = 0; + if (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0), + &local_errno) && + local_errno == EINVAL) { +# if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE) // Some older Android kernels have the tagged pointer ABI on // unconditionally, and hence don't have the tagged-addr prctl while still // allow the ABI. // If targeting Android and the prctl is not around we assume this is the // case. return; -#else - Printf( - "FATAL: " - "HWAddressSanitizer requires a kernel with tagged address ABI.\n"); - Die(); -#endif +# else + if (flags()->fail_without_syscall_abi) { + Printf( + "FATAL: " + "HWAddressSanitizer requires a kernel with tagged address ABI.\n"); + Die(); + } +# endif } // Turn on the tagged address ABI. - if (internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, 0, 0, 0) == - (uptr)-1 || - !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) { - Printf( - "FATAL: HWAddressSanitizer failed to enable tagged address syscall " - "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` " - "configuration.\n"); - Die(); + if ((internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL, + PR_TAGGED_ADDR_ENABLE, 0, 0, 0)) || + !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0))) { +# if defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE) + // Try the new prctl API for Intel LAM. The API is based on a currently + // unsubmitted patch to the Linux kernel (as of May 2021) and is thus + // subject to change. Patch is here: + // https://lore.kernel.org/linux-mm/20210205151631.43511-12-kirill.shutemov@linux.intel.com/ + int tag_bits = kTagBits; + int tag_shift = kAddressTagShift; + if (!internal_iserror( + internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, + reinterpret_cast<unsigned long>(&tag_bits), + reinterpret_cast<unsigned long>(&tag_shift), 0))) { + CHECK_EQ(tag_bits, kTagBits); + CHECK_EQ(tag_shift, kAddressTagShift); + return; + } +# endif // defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE) + if (flags()->fail_without_syscall_abi) { + Printf( + "FATAL: HWAddressSanitizer failed to enable tagged address syscall " + "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` " + "configuration.\n"); + Die(); + } } #undef PR_SET_TAGGED_ADDR_CTRL #undef PR_GET_TAGGED_ADDR_CTRL @@ -214,23 +232,16 @@ void InitThreads() { ProtectGap(thread_space_end, __hwasan_shadow_memory_dynamic_address - thread_space_end); InitThreadList(thread_space_start, thread_space_end - thread_space_start); + hwasanThreadList().CreateCurrentThread(); } bool MemIsApp(uptr p) { +// Memory outside the alias range has non-zero tags. +# if !defined(HWASAN_ALIASING_MODE) CHECK(GetTagFromPointer(p) == 0); - return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd); -} +# endif -static void HwasanAtExit(void) { - if (common_flags()->print_module_map) - DumpProcessMap(); - if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0)) - ReportStats(); - if (hwasan_report_count > 0) { - // ReportAtExitStatistics(); - if (common_flags()->exitcode) - internal__exit(common_flags()->exitcode); - } + return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd); } void InstallAtExitHandler() { @@ -309,22 +320,6 @@ void AndroidTestTlsSlot() { void AndroidTestTlsSlot() {} #endif -Thread *GetCurrentThread() { - uptr *ThreadLongPtr = GetCurrentThreadLongPtr(); - if (UNLIKELY(*ThreadLongPtr == 0)) - return nullptr; - auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr; - return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next()); -} - -struct AccessInfo { - uptr addr; - uptr size; - bool is_store; - bool is_load; - bool recover; -}; - static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { // Access type is passed in a platform dependent way (see below) and encoded // as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is @@ -375,28 +370,6 @@ static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { return AccessInfo{addr, size, is_store, !is_store, recover}; } -static void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, - ucontext_t *uc, uptr *registers_frame = nullptr) { - InternalMmapVector<BufferedStackTrace> stack_buffer(1); - BufferedStackTrace *stack = stack_buffer.data(); - stack->Reset(); - stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal); - - // The second stack frame contains the failure __hwasan_check function, as - // we have a stack frame for the registers saved in __hwasan_tag_mismatch that - // we wish to ignore. This (currently) only occurs on AArch64, as x64 - // implementations use SIGTRAP to implement the failure, and thus do not go - // through the stack saver. - if (registers_frame && stack->trace && stack->size > 0) { - stack->trace++; - stack->size--; - } - - bool fatal = flags()->halt_on_error || !ai.recover; - ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal, - registers_frame); -} - static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { AccessInfo ai = GetAccessInfo(info, uc); if (!ai.is_store && !ai.is_load) @@ -429,27 +402,39 @@ void HwasanOnDeadlySignal(int signo, void *info, void *context) { HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr); } +void Thread::InitStackAndTls(const InitState *) { + uptr tls_size; + uptr stack_size; + GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_, + &tls_size); + stack_top_ = stack_bottom_ + stack_size; + tls_end_ = tls_begin_ + tls_size; +} -} // namespace __hwasan - -// Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the -// rest of the mismatch handling code (C++). -void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame, - size_t outsize) { - __hwasan::AccessInfo ai; - ai.is_store = access_info & 0x10; - ai.is_load = !ai.is_store; - ai.recover = access_info & 0x20; - ai.addr = addr; - if ((access_info & 0xf) == 0xf) - ai.size = outsize; - else - ai.size = 1 << (access_info & 0xf); - - __hwasan::HandleTagMismatch(ai, (uptr)__builtin_return_address(0), - (uptr)__builtin_frame_address(0), nullptr, - registers_frame); - __builtin_unreachable(); +uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { + CHECK(IsAligned(p, kShadowAlignment)); + CHECK(IsAligned(size, kShadowAlignment)); + uptr shadow_start = MemToShadow(p); + uptr shadow_size = MemToShadowSize(size); + + uptr page_size = GetPageSizeCached(); + uptr page_start = RoundUpTo(shadow_start, page_size); + uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size); + uptr threshold = common_flags()->clear_shadow_mmap_threshold; + if (SANITIZER_LINUX && + UNLIKELY(page_end >= page_start + threshold && tag == 0)) { + internal_memset((void *)shadow_start, tag, page_start - shadow_start); + internal_memset((void *)page_end, tag, + shadow_start + shadow_size - page_end); + // For an anonymous private mapping MADV_DONTNEED will return a zero page on + // Linux. + ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end); + } else { + internal_memset((void *)shadow_start, tag, shadow_size); + } + return AddTagToPointer(p, tag); } +} // namespace __hwasan + #endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD diff --git a/compiler-rt/lib/hwasan/hwasan_mapping.h b/compiler-rt/lib/hwasan/hwasan_mapping.h index c149687bdfa6..79a143632f6a 100644 --- a/compiler-rt/lib/hwasan/hwasan_mapping.h +++ b/compiler-rt/lib/hwasan/hwasan_mapping.h @@ -48,12 +48,14 @@ extern uptr kHighShadowEnd; extern uptr kHighMemStart; extern uptr kHighMemEnd; +inline uptr GetShadowOffset() { + return SANITIZER_FUCHSIA ? 0 : __hwasan_shadow_memory_dynamic_address; +} inline uptr MemToShadow(uptr untagged_addr) { - return (untagged_addr >> kShadowScale) + - __hwasan_shadow_memory_dynamic_address; + return (untagged_addr >> kShadowScale) + GetShadowOffset(); } inline uptr ShadowToMem(uptr shadow_addr) { - return (shadow_addr - __hwasan_shadow_memory_dynamic_address) << kShadowScale; + return (shadow_addr - GetShadowOffset()) << kShadowScale; } inline uptr MemToShadowSize(uptr size) { return size >> kShadowScale; @@ -61,6 +63,13 @@ inline uptr MemToShadowSize(uptr size) { bool MemIsApp(uptr p); +inline bool MemIsShadow(uptr p) { + return (kLowShadowStart <= p && p <= kLowShadowEnd) || + (kHighShadowStart <= p && p <= kHighShadowEnd); +} + +uptr GetAliasRegionStart(); + } // namespace __hwasan #endif // HWASAN_MAPPING_H diff --git a/compiler-rt/lib/hwasan/hwasan_memintrinsics.cpp b/compiler-rt/lib/hwasan/hwasan_memintrinsics.cpp index e82d77a1bc16..fab017aae60b 100644 --- a/compiler-rt/lib/hwasan/hwasan_memintrinsics.cpp +++ b/compiler-rt/lib/hwasan/hwasan_memintrinsics.cpp @@ -24,7 +24,7 @@ using namespace __hwasan; void *__hwasan_memset(void *block, int c, uptr size) { CheckAddressSized<ErrorAction::Recover, AccessType::Store>( reinterpret_cast<uptr>(block), size); - return memset(UntagPtr(block), c, size); + return memset(block, c, size); } void *__hwasan_memcpy(void *to, const void *from, uptr size) { @@ -32,7 +32,7 @@ void *__hwasan_memcpy(void *to, const void *from, uptr size) { reinterpret_cast<uptr>(to), size); CheckAddressSized<ErrorAction::Recover, AccessType::Load>( reinterpret_cast<uptr>(from), size); - return memcpy(UntagPtr(to), UntagPtr(from), size); + return memcpy(to, from, size); } void *__hwasan_memmove(void *to, const void *from, uptr size) { diff --git a/compiler-rt/lib/hwasan/hwasan_new_delete.cpp b/compiler-rt/lib/hwasan/hwasan_new_delete.cpp index 8d01d3944f2b..4e057a651e1d 100644 --- a/compiler-rt/lib/hwasan/hwasan_new_delete.cpp +++ b/compiler-rt/lib/hwasan/hwasan_new_delete.cpp @@ -27,6 +27,12 @@ void *res = hwasan_malloc(size, &stack);\ if (!nothrow && UNLIKELY(!res)) ReportOutOfMemory(size, &stack);\ return res +#define OPERATOR_NEW_ALIGN_BODY(nothrow) \ + GET_MALLOC_STACK_TRACE; \ + void *res = hwasan_aligned_alloc(static_cast<uptr>(align), size, &stack); \ + if (!nothrow && UNLIKELY(!res)) \ + ReportOutOfMemory(size, &stack); \ + return res #define OPERATOR_DELETE_BODY \ GET_MALLOC_STACK_TRACE; \ @@ -67,15 +73,63 @@ void *operator new[](size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY(true /*nothrow*/); } -INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } -INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } -INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; } -INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -void operator delete[](void *ptr, std::nothrow_t const&) { +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(void *ptr) + NOEXCEPT { + OPERATOR_DELETE_BODY; +} +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( + void *ptr) NOEXCEPT { + OPERATOR_DELETE_BODY; +} +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( + void *ptr, std::nothrow_t const &) { + OPERATOR_DELETE_BODY; +} +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( + void *ptr, std::nothrow_t const &) { + OPERATOR_DELETE_BODY; +} + +#endif // OPERATOR_NEW_BODY + +#ifdef OPERATOR_NEW_ALIGN_BODY + +namespace std { +enum class align_val_t : size_t {}; +} // namespace std + +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new( + size_t size, std::align_val_t align) { + OPERATOR_NEW_ALIGN_BODY(false /*nothrow*/); +} +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[]( + size_t size, std::align_val_t align) { + OPERATOR_NEW_ALIGN_BODY(false /*nothrow*/); +} +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new( + size_t size, std::align_val_t align, std::nothrow_t const &) { + OPERATOR_NEW_ALIGN_BODY(true /*nothrow*/); +} +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[]( + size_t size, std::align_val_t align, std::nothrow_t const &) { + OPERATOR_NEW_ALIGN_BODY(true /*nothrow*/); +} + +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( + void *ptr, std::align_val_t align) NOEXCEPT { + OPERATOR_DELETE_BODY; +} +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( + void *ptr, std::align_val_t) NOEXCEPT { + OPERATOR_DELETE_BODY; +} +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( + void *ptr, std::align_val_t, std::nothrow_t const &) NOEXCEPT { + OPERATOR_DELETE_BODY; +} +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( + void *ptr, std::align_val_t, std::nothrow_t const &) NOEXCEPT { OPERATOR_DELETE_BODY; } -#endif // OPERATOR_NEW_BODY +#endif // OPERATOR_NEW_ALIGN_BODY diff --git a/compiler-rt/lib/hwasan/hwasan_poisoning.cpp b/compiler-rt/lib/hwasan/hwasan_poisoning.cpp index 2a0816428e75..5aafdb1884b5 100644 --- a/compiler-rt/lib/hwasan/hwasan_poisoning.cpp +++ b/compiler-rt/lib/hwasan/hwasan_poisoning.cpp @@ -19,30 +19,6 @@ namespace __hwasan { -uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { - CHECK(IsAligned(p, kShadowAlignment)); - CHECK(IsAligned(size, kShadowAlignment)); - uptr shadow_start = MemToShadow(p); - uptr shadow_size = MemToShadowSize(size); - - uptr page_size = GetPageSizeCached(); - uptr page_start = RoundUpTo(shadow_start, page_size); - uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size); - uptr threshold = common_flags()->clear_shadow_mmap_threshold; - if (SANITIZER_LINUX && - UNLIKELY(page_end >= page_start + threshold && tag == 0)) { - internal_memset((void *)shadow_start, tag, page_start - shadow_start); - internal_memset((void *)page_end, tag, - shadow_start + shadow_size - page_end); - // For an anonymous private mapping MADV_DONTNEED will return a zero page on - // Linux. - ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end); - } else { - internal_memset((void *)shadow_start, tag, shadow_size); - } - return AddTagToPointer(p, tag); -} - uptr TagMemory(uptr p, uptr size, tag_t tag) { uptr start = RoundDownTo(p, kShadowAlignment); uptr end = RoundUpTo(p + size, kShadowAlignment); diff --git a/compiler-rt/lib/hwasan/hwasan_report.cpp b/compiler-rt/lib/hwasan/hwasan_report.cpp index 4448d9243767..44047c9fdaf8 100644 --- a/compiler-rt/lib/hwasan/hwasan_report.cpp +++ b/compiler-rt/lib/hwasan/hwasan_report.cpp @@ -224,7 +224,7 @@ static void PrintStackAllocations(StackAllocationsRingBuffer *sa, // We didn't find any locals. Most likely we don't have symbols, so dump // the information that we have for offline analysis. - InternalScopedString frame_desc(GetPageSizeCached() * 2); + InternalScopedString frame_desc; Printf("Previously allocated frames:\n"); for (uptr i = 0; i < frames; i++) { const uptr *record_addr = &(*sa)[i]; @@ -236,12 +236,12 @@ static void PrintStackAllocations(StackAllocationsRingBuffer *sa, frame_desc.append(" record_addr:0x%zx record:0x%zx", reinterpret_cast<uptr>(record_addr), record); if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) { - RenderFrame(&frame_desc, " %F %L\n", 0, frame->info.address, &frame->info, + RenderFrame(&frame_desc, " %F %L", 0, frame->info.address, &frame->info, common_flags()->symbolize_vs_style, common_flags()->strip_path_prefix); frame->ClearAll(); } - Printf("%s", frame_desc.data()); + Printf("%s\n", frame_desc.data()); frame_desc.clear(); } } @@ -296,6 +296,75 @@ static uptr GetGlobalSizeFromDescriptor(uptr ptr) { return 0; } +static void ShowHeapOrGlobalCandidate(uptr untagged_addr, tag_t *candidate, + tag_t *left, tag_t *right) { + Decorator d; + uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate)); + HwasanChunkView chunk = FindHeapChunkByAddress(mem); + if (chunk.IsAllocated()) { + uptr offset; + const char *whence; + if (untagged_addr < chunk.End() && untagged_addr >= chunk.Beg()) { + offset = untagged_addr - chunk.Beg(); + whence = "inside"; + } else if (candidate == left) { + offset = untagged_addr - chunk.End(); + whence = "to the right of"; + } else { + offset = chunk.Beg() - untagged_addr; + whence = "to the left of"; + } + Printf("%s", d.Error()); + Printf("\nCause: heap-buffer-overflow\n"); + Printf("%s", d.Default()); + Printf("%s", d.Location()); + Printf("%p is located %zd bytes %s %zd-byte region [%p,%p)\n", + untagged_addr, offset, whence, chunk.UsedSize(), chunk.Beg(), + chunk.End()); + Printf("%s", d.Allocation()); + Printf("allocated here:\n"); + Printf("%s", d.Default()); + GetStackTraceFromId(chunk.GetAllocStackId()).Print(); + return; + } + // Check whether the address points into a loaded library. If so, this is + // most likely a global variable. + const char *module_name; + uptr module_address; + Symbolizer *sym = Symbolizer::GetOrInit(); + if (sym->GetModuleNameAndOffsetForPC(mem, &module_name, &module_address)) { + Printf("%s", d.Error()); + Printf("\nCause: global-overflow\n"); + Printf("%s", d.Default()); + DataInfo info; + Printf("%s", d.Location()); + if (sym->SymbolizeData(mem, &info) && info.start) { + Printf( + "%p is located %zd bytes to the %s of %zd-byte global variable " + "%s [%p,%p) in %s\n", + untagged_addr, + candidate == left ? untagged_addr - (info.start + info.size) + : info.start - untagged_addr, + candidate == left ? "right" : "left", info.size, info.name, + info.start, info.start + info.size, module_name); + } else { + uptr size = GetGlobalSizeFromDescriptor(mem); + if (size == 0) + // We couldn't find the size of the global from the descriptors. + Printf("%p is located to the %s of a global variable in (%s+0x%x)\n", + untagged_addr, candidate == left ? "right" : "left", module_name, + module_address); + else + Printf( + "%p is located to the %s of a %zd-byte global variable in " + "(%s+0x%x)\n", + untagged_addr, candidate == left ? "right" : "left", size, + module_name, module_address); + } + Printf("%s", d.Default()); + } +} + void PrintAddressDescription( uptr tagged_addr, uptr access_size, StackAllocationsRingBuffer *current_stack_allocations) { @@ -317,78 +386,59 @@ void PrintAddressDescription( d.Default()); } + tag_t addr_tag = GetTagFromPointer(tagged_addr); + + bool on_stack = false; + // Check stack first. If the address is on the stack of a live thread, we + // know it cannot be a heap / global overflow. + hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { + if (t->AddrIsInStack(untagged_addr)) { + on_stack = true; + // TODO(fmayer): figure out how to distinguish use-after-return and + // stack-buffer-overflow. + Printf("%s", d.Error()); + Printf("\nCause: stack tag-mismatch\n"); + Printf("%s", d.Location()); + Printf("Address %p is located in stack of thread T%zd\n", untagged_addr, + t->unique_id()); + Printf("%s", d.Default()); + t->Announce(); + + auto *sa = (t == GetCurrentThread() && current_stack_allocations) + ? current_stack_allocations + : t->stack_allocations(); + PrintStackAllocations(sa, addr_tag, untagged_addr); + num_descriptions_printed++; + } + }); + // Check if this looks like a heap buffer overflow by scanning // the shadow left and right and looking for the first adjacent // object with a different memory tag. If that tag matches addr_tag, // check the allocator if it has a live chunk there. - tag_t addr_tag = GetTagFromPointer(tagged_addr); tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr)); tag_t *candidate = nullptr, *left = tag_ptr, *right = tag_ptr; - for (int i = 0; i < 1000; i++) { - if (TagsEqual(addr_tag, left)) { + uptr candidate_distance = 0; + for (; candidate_distance < 1000; candidate_distance++) { + if (MemIsShadow(reinterpret_cast<uptr>(left)) && + TagsEqual(addr_tag, left)) { candidate = left; break; } --left; - if (TagsEqual(addr_tag, right)) { + if (MemIsShadow(reinterpret_cast<uptr>(right)) && + TagsEqual(addr_tag, right)) { candidate = right; break; } ++right; } - if (candidate) { - uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate)); - HwasanChunkView chunk = FindHeapChunkByAddress(mem); - if (chunk.IsAllocated()) { - Printf("%s", d.Location()); - Printf("%p is located %zd bytes to the %s of %zd-byte region [%p,%p)\n", - untagged_addr, - candidate == left ? untagged_addr - chunk.End() - : chunk.Beg() - untagged_addr, - candidate == left ? "right" : "left", chunk.UsedSize(), - chunk.Beg(), chunk.End()); - Printf("%s", d.Allocation()); - Printf("allocated here:\n"); - Printf("%s", d.Default()); - GetStackTraceFromId(chunk.GetAllocStackId()).Print(); - num_descriptions_printed++; - } else { - // Check whether the address points into a loaded library. If so, this is - // most likely a global variable. - const char *module_name; - uptr module_address; - Symbolizer *sym = Symbolizer::GetOrInit(); - if (sym->GetModuleNameAndOffsetForPC(mem, &module_name, - &module_address)) { - DataInfo info; - if (sym->SymbolizeData(mem, &info) && info.start) { - Printf( - "%p is located %zd bytes to the %s of %zd-byte global variable " - "%s [%p,%p) in %s\n", - untagged_addr, - candidate == left ? untagged_addr - (info.start + info.size) - : info.start - untagged_addr, - candidate == left ? "right" : "left", info.size, info.name, - info.start, info.start + info.size, module_name); - } else { - uptr size = GetGlobalSizeFromDescriptor(mem); - if (size == 0) - // We couldn't find the size of the global from the descriptors. - Printf( - "%p is located to the %s of a global variable in (%s+0x%x)\n", - untagged_addr, candidate == left ? "right" : "left", - module_name, module_address); - else - Printf( - "%p is located to the %s of a %zd-byte global variable in " - "(%s+0x%x)\n", - untagged_addr, candidate == left ? "right" : "left", size, - module_name, module_address); - } - num_descriptions_printed++; - } - } + constexpr auto kCloseCandidateDistance = 1; + + if (!on_stack && candidate && candidate_distance <= kCloseCandidateDistance) { + ShowHeapOrGlobalCandidate(untagged_addr, candidate, left, right); + num_descriptions_printed++; } hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { @@ -398,6 +448,8 @@ void PrintAddressDescription( if (FindHeapAllocation(t->heap_allocations(), tagged_addr, &har, &ring_index, &num_matching_addrs, &num_matching_addrs_4b)) { + Printf("%s", d.Error()); + Printf("\nCause: use-after-free\n"); Printf("%s", d.Location()); Printf("%p is located %zd bytes inside of %zd-byte region [%p,%p)\n", untagged_addr, untagged_addr - UntagAddr(har.tagged_addr), @@ -424,29 +476,25 @@ void PrintAddressDescription( t->Announce(); num_descriptions_printed++; } - - // Very basic check for stack memory. - if (t->AddrIsInStack(untagged_addr)) { - Printf("%s", d.Location()); - Printf("Address %p is located in stack of thread T%zd\n", untagged_addr, - t->unique_id()); - Printf("%s", d.Default()); - t->Announce(); - - auto *sa = (t == GetCurrentThread() && current_stack_allocations) - ? current_stack_allocations - : t->stack_allocations(); - PrintStackAllocations(sa, addr_tag, untagged_addr); - num_descriptions_printed++; - } }); + if (candidate && num_descriptions_printed == 0) { + ShowHeapOrGlobalCandidate(untagged_addr, candidate, left, right); + num_descriptions_printed++; + } + // Print the remaining threads, as an extra information, 1 line per thread. hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { t->Announce(); }); if (!num_descriptions_printed) // We exhausted our possibilities. Bail out. Printf("HWAddressSanitizer can not describe address in more detail.\n"); + if (num_descriptions_printed > 1) { + Printf( + "There are %d potential causes, printed above in order " + "of likeliness.\n", + num_descriptions_printed); + } } void ReportStats() {} @@ -459,7 +507,7 @@ static void PrintTagInfoAroundAddr(tag_t *tag_ptr, uptr num_rows, RoundDownTo(reinterpret_cast<uptr>(tag_ptr), row_len)); tag_t *beg_row = center_row_beg - row_len * (num_rows / 2); tag_t *end_row = center_row_beg + row_len * ((num_rows + 1) / 2); - InternalScopedString s(GetPageSizeCached() * 8); + InternalScopedString s; for (tag_t *row = beg_row; row < end_row; row += row_len) { s.append("%s", row == center_row_beg ? "=>" : " "); s.append("%p:", row); @@ -538,6 +586,12 @@ void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, Report("ERROR: %s: %s; heap object [%p,%p) of size %zd\n", SanitizerToolName, bug_type, untagged_addr, untagged_addr + orig_size, orig_size); Printf("\n%s", d.Default()); + Printf( + "Stack of invalid access unknown. Issue detected at deallocation " + "time.\n"); + Printf("%s", d.Allocation()); + Printf("deallocated here:\n"); + Printf("%s", d.Default()); stack->Print(); HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr); if (chunk.Beg()) { @@ -547,7 +601,7 @@ void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, GetStackTraceFromId(chunk.GetAllocStackId()).Print(); } - InternalScopedString s(GetPageSizeCached() * 8); + InternalScopedString s; CHECK_GT(tail_size, 0U); CHECK_LT(tail_size, kShadowAlignment); u8 *tail = reinterpret_cast<u8*>(untagged_addr + orig_size); @@ -657,8 +711,10 @@ void ReportRegisters(uptr *frame, uptr pc) { frame[20], frame[21], frame[22], frame[23]); Printf(" x24 %016llx x25 %016llx x26 %016llx x27 %016llx\n", frame[24], frame[25], frame[26], frame[27]); - Printf(" x28 %016llx x29 %016llx x30 %016llx\n", - frame[28], frame[29], frame[30]); + // hwasan_check* reduces the stack pointer by 256, then __hwasan_tag_mismatch + // passes it to this function. + Printf(" x28 %016llx x29 %016llx x30 %016llx sp %016llx\n", frame[28], + frame[29], frame[30], reinterpret_cast<u8 *>(frame) + 256); } } // namespace __hwasan diff --git a/compiler-rt/lib/hwasan/hwasan_setjmp.S b/compiler-rt/lib/hwasan/hwasan_setjmp.S index 0c1354331940..381af63363cc 100644 --- a/compiler-rt/lib/hwasan/hwasan_setjmp.S +++ b/compiler-rt/lib/hwasan/hwasan_setjmp.S @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_asm.h" +#include "builtins/assembly.h" #if HWASAN_WITH_INTERCEPTORS && defined(__aarch64__) #include "sanitizer_common/sanitizer_platform.h" @@ -34,6 +35,7 @@ ASM_TYPE_FUNCTION(__interceptor_setjmp) __interceptor_setjmp: CFI_STARTPROC + BTI_C mov x1, #0 b __interceptor_sigsetjmp CFI_ENDPROC @@ -46,6 +48,7 @@ ASM_SIZE(__interceptor_setjmp) ASM_TYPE_FUNCTION(__interceptor_setjmp_bionic) __interceptor_setjmp_bionic: CFI_STARTPROC + BTI_C mov x1, #1 b __interceptor_sigsetjmp CFI_ENDPROC @@ -56,6 +59,7 @@ ASM_SIZE(__interceptor_setjmp_bionic) ASM_TYPE_FUNCTION(__interceptor_sigsetjmp) __interceptor_sigsetjmp: CFI_STARTPROC + BTI_C stp x19, x20, [x0, #0<<3] stp x21, x22, [x0, #2<<3] stp x23, x24, [x0, #4<<3] @@ -98,3 +102,5 @@ ALIAS __interceptor_setjmp, _setjmp // We do not need executable stack. NO_EXEC_STACK_DIRECTIVE + +GNU_PROPERTY_BTI_PAC diff --git a/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S index 08df12736bb4..bcb0df420190 100644 --- a/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S +++ b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S @@ -1,4 +1,5 @@ #include "sanitizer_common/sanitizer_asm.h" +#include "builtins/assembly.h" // The content of this file is AArch64-only: #if defined(__aarch64__) @@ -74,6 +75,8 @@ .global __hwasan_tag_mismatch .type __hwasan_tag_mismatch, %function __hwasan_tag_mismatch: + BTI_J + // Compute the granule position one past the end of the access. mov x16, #1 and x17, x1, #0xf @@ -106,6 +109,7 @@ __hwasan_tag_mismatch: .type __hwasan_tag_mismatch_v2, %function __hwasan_tag_mismatch_v2: CFI_STARTPROC + BTI_J // Set the CFA to be the return address for caller of __hwasan_check_*. Note // that we do not emit CFI predicates to describe the contents of this stack @@ -150,3 +154,5 @@ __hwasan_tag_mismatch_v2: // We do not need executable stack. NO_EXEC_STACK_DIRECTIVE + +GNU_PROPERTY_BTI_PAC diff --git a/compiler-rt/lib/hwasan/hwasan_thread.cpp b/compiler-rt/lib/hwasan/hwasan_thread.cpp index b81a6350c05c..ee747a3beea5 100644 --- a/compiler-rt/lib/hwasan/hwasan_thread.cpp +++ b/compiler-rt/lib/hwasan/hwasan_thread.cpp @@ -34,12 +34,28 @@ void Thread::InitRandomState() { stack_allocations_->push(0); } -void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) { +void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size, + const InitState *state) { + CHECK_EQ(0, unique_id_); // try to catch bad stack reuse + CHECK_EQ(0, stack_top_); + CHECK_EQ(0, stack_bottom_); + static u64 unique_id; unique_id_ = unique_id++; if (auto sz = flags()->heap_history_size) heap_allocations_ = HeapAllocationsRingBuffer::New(sz); + InitStackAndTls(state); +#if !SANITIZER_FUCHSIA + // Do not initialize the stack ring buffer just yet on Fuchsia. Threads will + // be initialized before we enter the thread itself, so we will instead call + // this later. + InitStackRingBuffer(stack_buffer_start, stack_buffer_size); +#endif +} + +void Thread::InitStackRingBuffer(uptr stack_buffer_start, + uptr stack_buffer_size) { HwasanTSDThreadInit(); // Only needed with interceptors. uptr *ThreadLong = GetCurrentThreadLongPtr(); // The following implicitly sets (this) as the current thread. @@ -51,13 +67,6 @@ void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) { // ScopedTaggingDisable needs GetCurrentThread to be set up. ScopedTaggingDisabler disabler; - uptr tls_size; - uptr stack_size; - GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_, - &tls_size); - stack_top_ = stack_bottom_ + stack_size; - tls_end_ = tls_begin_ + tls_size; - if (stack_bottom_) { int local; CHECK(AddrIsInStack((uptr)&local)); @@ -113,18 +122,21 @@ static u32 xorshift(u32 state) { } // Generate a (pseudo-)random non-zero tag. -tag_t Thread::GenerateRandomTag() { +tag_t Thread::GenerateRandomTag(uptr num_bits) { + DCHECK_GT(num_bits, 0); if (tagging_disabled_) return 0; tag_t tag; + const uptr tag_mask = (1ULL << num_bits) - 1; do { if (flags()->random_tags) { if (!random_buffer_) random_buffer_ = random_state_ = xorshift(random_state_); CHECK(random_buffer_); - tag = random_buffer_ & 0xFF; - random_buffer_ >>= 8; + tag = random_buffer_ & tag_mask; + random_buffer_ >>= num_bits; } else { - tag = random_state_ = (random_state_ + 1) & 0xFF; + random_state_ += 1; + tag = random_state_ & tag_mask; } } while (!tag); return tag; diff --git a/compiler-rt/lib/hwasan/hwasan_thread.h b/compiler-rt/lib/hwasan/hwasan_thread.h index 88958daf767c..9f20afe1dc76 100644 --- a/compiler-rt/lib/hwasan/hwasan_thread.h +++ b/compiler-rt/lib/hwasan/hwasan_thread.h @@ -23,8 +23,17 @@ typedef __sanitizer::CompactRingBuffer<uptr> StackAllocationsRingBuffer; class Thread { public: - void Init(uptr stack_buffer_start, uptr stack_buffer_size); // Must be called from the thread itself. + // These are optional parameters that can be passed to Init. + struct InitState; + + void Init(uptr stack_buffer_start, uptr stack_buffer_size, + const InitState *state = nullptr); void InitRandomState(); + void InitStackAndTls(const InitState *state = nullptr); + + // Must be called from the thread itself. + void InitStackRingBuffer(uptr stack_buffer_start, uptr stack_buffer_size); + void Destroy(); uptr stack_top() { return stack_top_; } @@ -42,7 +51,7 @@ class Thread { HeapAllocationsRingBuffer *heap_allocations() { return heap_allocations_; } StackAllocationsRingBuffer *stack_allocations() { return stack_allocations_; } - tag_t GenerateRandomTag(); + tag_t GenerateRandomTag(uptr num_bits = kTagBits); void DisableTagging() { tagging_disabled_++; } void EnableTagging() { tagging_disabled_--; } diff --git a/compiler-rt/lib/hwasan/hwasan_thread_list.cpp b/compiler-rt/lib/hwasan/hwasan_thread_list.cpp index a31eee84ed93..fa46e658b69d 100644 --- a/compiler-rt/lib/hwasan/hwasan_thread_list.cpp +++ b/compiler-rt/lib/hwasan/hwasan_thread_list.cpp @@ -12,4 +12,4 @@ void InitThreadList(uptr storage, uptr size) { new (thread_list_placeholder) HwasanThreadList(storage, size); } -} // namespace +} // namespace __hwasan diff --git a/compiler-rt/lib/hwasan/hwasan_thread_list.h b/compiler-rt/lib/hwasan/hwasan_thread_list.h index e596bde36662..15916a802d6e 100644 --- a/compiler-rt/lib/hwasan/hwasan_thread_list.h +++ b/compiler-rt/lib/hwasan/hwasan_thread_list.h @@ -85,21 +85,26 @@ class HwasanThreadList { RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2); } - Thread *CreateCurrentThread() { - Thread *t; + Thread *CreateCurrentThread(const Thread::InitState *state = nullptr) { + Thread *t = nullptr; { - SpinMutexLock l(&list_mutex_); + SpinMutexLock l(&free_list_mutex_); if (!free_list_.empty()) { t = free_list_.back(); free_list_.pop_back(); - uptr start = (uptr)t - ring_buffer_size_; - internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread)); - } else { - t = AllocThread(); } + } + if (t) { + uptr start = (uptr)t - ring_buffer_size_; + internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread)); + } else { + t = AllocThread(); + } + { + SpinMutexLock l(&live_list_mutex_); live_list_.push_back(t); } - t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_); + t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_, state); AddThreadStats(t); return t; } @@ -110,6 +115,7 @@ class HwasanThreadList { } void RemoveThreadFromLiveList(Thread *t) { + SpinMutexLock l(&live_list_mutex_); for (Thread *&t2 : live_list_) if (t2 == t) { // To remove t2, copy the last element of the list in t2's position, and @@ -124,10 +130,10 @@ class HwasanThreadList { void ReleaseThread(Thread *t) { RemoveThreadStats(t); t->Destroy(); - SpinMutexLock l(&list_mutex_); + DontNeedThread(t); RemoveThreadFromLiveList(t); + SpinMutexLock l(&free_list_mutex_); free_list_.push_back(t); - DontNeedThread(t); } Thread *GetThreadByBufferAddress(uptr p) { @@ -144,7 +150,7 @@ class HwasanThreadList { template <class CB> void VisitAllLiveThreads(CB cb) { - SpinMutexLock l(&list_mutex_); + SpinMutexLock l(&live_list_mutex_); for (Thread *t : live_list_) cb(t); } @@ -165,8 +171,11 @@ class HwasanThreadList { return stats_; } + uptr GetRingBufferSize() const { return ring_buffer_size_; } + private: Thread *AllocThread() { + SpinMutexLock l(&free_space_mutex_); uptr align = ring_buffer_size_ * 2; CHECK(IsAligned(free_space_, align)); Thread *t = (Thread *)(free_space_ + ring_buffer_size_); @@ -175,14 +184,16 @@ class HwasanThreadList { return t; } + SpinMutex free_space_mutex_; uptr free_space_; uptr free_space_end_; uptr ring_buffer_size_; uptr thread_alloc_size_; + SpinMutex free_list_mutex_; InternalMmapVector<Thread *> free_list_; + SpinMutex live_list_mutex_; InternalMmapVector<Thread *> live_list_; - SpinMutex list_mutex_; ThreadStats stats_; SpinMutex stats_mutex_; @@ -191,4 +202,4 @@ class HwasanThreadList { void InitThreadList(uptr storage, uptr size); HwasanThreadList &hwasanThreadList(); -} // namespace +} // namespace __hwasan |