Vendor import of llvm-project main 88e66fa60ae5, the last commit beforevendor/llvm-project/llvmorg-13-init-16847-g88e66fa60ae5vendor/llvm-project/llvmorg-12.0.1-rc2-0-ge7dac564cd0evendor/llvm-project/llvmorg-12.0.1-0-gfed41342a82f

the upstream release/13.x branch was created.

26 files changed, 1010 insertions, 468 deletions

diff --git a/compiler-rt/lib/hwasan/hwasan.cpp b/compiler-rt/lib/hwasan/hwasan.cpp
index c5322110cb66..cbe0dee66dcd 100644
--- a/compiler-rt/lib/hwasan/hwasan.cpp
+++ b/compiler-rt/lib/hwasan/hwasan.cpp
@@ -50,6 +50,11 @@ bool hwasan_init_is_running;
 
 int hwasan_report_count = 0;
 
+uptr kLowShadowStart;
+uptr kLowShadowEnd;
+uptr kHighShadowStart;
+uptr kHighShadowEnd;
+
 void Flags::SetDefaults() {
 #define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
 #include "hwasan_flags.inc"
@@ -128,16 +133,11 @@ static void InitializeFlags() {
   if (common_flags()->help) parser.PrintFlagDescriptions();
 }
 
-static void HWAsanCheckFailed(const char *file, int line, const char *cond,
-                              u64 v1, u64 v2) {
-  Report("HWAddressSanitizer CHECK failed: %s:%d \"%s\" (0x%zx, 0x%zx)\n", file,
-         line, cond, (uptr)v1, (uptr)v2);
-  PRINT_CURRENT_STACK_CHECK();
-  Die();
+static void CheckUnwind() {
+  GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
+  stack.Print();
 }
 
-static constexpr uptr kMemoryUsageBufferSize = 4096;
-
 static void HwasanFormatMemoryUsage(InternalScopedString &s) {
   HwasanThreadList &thread_list = hwasanThreadList();
   auto thread_stats = thread_list.GetThreadStats();
@@ -155,6 +155,8 @@ static void HwasanFormatMemoryUsage(InternalScopedString &s) {
 }
 
 #if SANITIZER_ANDROID
+static constexpr uptr kMemoryUsageBufferSize = 4096;
+
 static char *memory_usage_buffer = nullptr;
 
 static void InitMemoryUsage() {
@@ -171,7 +173,7 @@ void UpdateMemoryUsage() {
     return;
   if (!memory_usage_buffer)
     InitMemoryUsage();
-  InternalScopedString s(kMemoryUsageBufferSize);
+  InternalScopedString s;
   HwasanFormatMemoryUsage(s);
   internal_strncpy(memory_usage_buffer, s.data(), kMemoryUsageBufferSize - 1);
   memory_usage_buffer[kMemoryUsageBufferSize - 1] = '\0';
@@ -180,6 +182,65 @@ void UpdateMemoryUsage() {
 void UpdateMemoryUsage() {}
 #endif
 
+void HwasanAtExit() {
+  if (common_flags()->print_module_map)
+    DumpProcessMap();
+  if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
+    ReportStats();
+  if (hwasan_report_count > 0) {
+    // ReportAtExitStatistics();
+    if (common_flags()->exitcode)
+      internal__exit(common_flags()->exitcode);
+  }
+}
+
+void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, void *uc,
+                       uptr *registers_frame) {
+  InternalMmapVector<BufferedStackTrace> stack_buffer(1);
+  BufferedStackTrace *stack = stack_buffer.data();
+  stack->Reset();
+  stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal);
+
+  // The second stack frame contains the failure __hwasan_check function, as
+  // we have a stack frame for the registers saved in __hwasan_tag_mismatch that
+  // we wish to ignore. This (currently) only occurs on AArch64, as x64
+  // implementations use SIGTRAP to implement the failure, and thus do not go
+  // through the stack saver.
+  if (registers_frame && stack->trace && stack->size > 0) {
+    stack->trace++;
+    stack->size--;
+  }
+
+  bool fatal = flags()->halt_on_error || !ai.recover;
+  ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal,
+                    registers_frame);
+}
+
+void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame,
+                       size_t outsize) {
+  __hwasan::AccessInfo ai;
+  ai.is_store = access_info & 0x10;
+  ai.is_load = !ai.is_store;
+  ai.recover = access_info & 0x20;
+  ai.addr = addr;
+  if ((access_info & 0xf) == 0xf)
+    ai.size = outsize;
+  else
+    ai.size = 1 << (access_info & 0xf);
+
+  HandleTagMismatch(ai, (uptr)__builtin_return_address(0),
+                    (uptr)__builtin_frame_address(0), nullptr, registers_frame);
+  __builtin_unreachable();
+}
+
+Thread *GetCurrentThread() {
+  uptr *ThreadLongPtr = GetCurrentThreadLongPtr();
+  if (UNLIKELY(*ThreadLongPtr == 0))
+    return nullptr;
+  auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr;
+  return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next());
+}
+
 } // namespace __hwasan
 
 using namespace __hwasan;
@@ -219,7 +280,7 @@ static void InitLoadedGlobals() {
 static void InitInstrumentation() {
   if (hwasan_instrumentation_inited) return;
 
-  InitPrctl();
+  InitializeOsSupport();
 
   if (!InitShadow()) {
     Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n");
@@ -228,7 +289,6 @@ static void InitInstrumentation() {
   }
 
   InitThreads();
-  hwasanThreadList().CreateCurrentThread();
 
   hwasan_instrumentation_inited = 1;
 }
@@ -271,7 +331,7 @@ void __hwasan_init() {
   InitializeFlags();
 
   // Install tool-specific callbacks in sanitizer_common.
-  SetCheckFailedCallback(HWAsanCheckFailed);
+  SetCheckUnwindCallback(CheckUnwind);
 
   __sanitizer_set_report_path(common_flags()->log_path);
 
@@ -493,12 +553,12 @@ extern "C" void *__hwasan_extra_spill_area() {
 }
 
 void __hwasan_print_memory_usage() {
-  InternalScopedString s(kMemoryUsageBufferSize);
+  InternalScopedString s;
   HwasanFormatMemoryUsage(s);
   Printf("%s\n", s.data());
 }
 
-static const u8 kFallbackTag = 0xBB;
+static const u8 kFallbackTag = 0xBB & kTagMask;
 
 u8 __hwasan_generate_tag() {
   Thread *t = GetCurrentThread();
@@ -519,4 +579,12 @@ void __sanitizer_print_stack_trace() {
   GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
   stack.Print();
 }
+
+// Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the
+// rest of the mismatch handling code (C++).
+void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame,
+                            size_t outsize) {
+  __hwasan::HwasanTagMismatch(addr, access_info, registers_frame, outsize);
+}
+
 } // extern "C"
diff --git a/compiler-rt/lib/hwasan/hwasan.h b/compiler-rt/lib/hwasan/hwasan.h
index d4521efd089a..7338b696ad34 100644
--- a/compiler-rt/lib/hwasan/hwasan.h
+++ b/compiler-rt/lib/hwasan/hwasan.h
@@ -14,11 +14,12 @@
 #ifndef HWASAN_H
 #define HWASAN_H
 
+#include "hwasan_flags.h"
+#include "hwasan_interface_internal.h"
+#include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_stacktrace.h"
-#include "hwasan_interface_internal.h"
-#include "hwasan_flags.h"
 #include "ubsan/ubsan_platform.h"
 
 #ifndef HWASAN_CONTAINS_UBSAN
@@ -35,10 +36,38 @@
 
 typedef u8 tag_t;
 
+#if defined(HWASAN_ALIASING_MODE)
+#  if !defined(__x86_64__)
+#    error Aliasing mode is only supported on x86_64
+#  endif
+// Tags are done in middle bits using userspace aliasing.
+constexpr unsigned kAddressTagShift = 39;
+constexpr unsigned kTagBits = 3;
+
+// The alias region is placed next to the shadow so the upper bits of all
+// taggable addresses matches the upper bits of the shadow base.  This shift
+// value determines which upper bits must match.  It has a floor of 44 since the
+// shadow is always 8TB.
+// TODO(morehouse): In alias mode we can shrink the shadow and use a
+// simpler/faster shadow calculation.
+constexpr unsigned kTaggableRegionCheckShift =
+    __sanitizer::Max(kAddressTagShift + kTagBits + 1U, 44U);
+#elif defined(__x86_64__)
+// Tags are done in upper bits using Intel LAM.
+constexpr unsigned kAddressTagShift = 57;
+constexpr unsigned kTagBits = 6;
+#else
 // TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address
 // translation and can be used to store a tag.
-const unsigned kAddressTagShift = 56;
-const uptr kAddressTagMask = 0xFFUL << kAddressTagShift;
+constexpr unsigned kAddressTagShift = 56;
+constexpr unsigned kTagBits = 8;
+#endif  // defined(HWASAN_ALIASING_MODE)
+
+// Mask for extracting tag bits from the lower 8 bits.
+constexpr uptr kTagMask = (1UL << kTagBits) - 1;
+
+// Mask for extracting tag bits from full pointers.
+constexpr uptr kAddressTagMask = kTagMask << kAddressTagShift;
 
 // Minimal alignment of the shadow base address. Determines the space available
 // for threads and stack histories. This is an ABI constant.
@@ -50,7 +79,7 @@ const unsigned kRecordFPLShift = 4;
 const unsigned kRecordFPModulus = 1 << (64 - kRecordFPShift + kRecordFPLShift);
 
 static inline tag_t GetTagFromPointer(uptr p) {
-  return p >> kAddressTagShift;
+  return (p >> kAddressTagShift) & kTagMask;
 }
 
 static inline uptr UntagAddr(uptr tagged_addr) {
@@ -73,7 +102,7 @@ extern bool hwasan_init_is_running;
 extern int hwasan_report_count;
 
 bool InitShadow();
-void InitPrctl();
+void InitializeOsSupport();
 void InitThreads();
 void InitializeInterceptors();
 
@@ -105,17 +134,9 @@ void InstallAtExitHandler();
   if (hwasan_inited)                                     \
     stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal)
 
-#define GET_FATAL_STACK_TRACE_HERE \
-  GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME())
-
-#define PRINT_CURRENT_STACK_CHECK() \
-  {                                 \
-    GET_FATAL_STACK_TRACE_HERE;     \
-    stack.Print();                  \
-  }
-
 void HwasanTSDInit();
 void HwasanTSDThreadInit();
+void HwasanAtExit();
 
 void HwasanOnDeadlySignal(int signo, void *info, void *context);
 
@@ -125,6 +146,26 @@ void AppendToErrorMessageBuffer(const char *buffer);
 
 void AndroidTestTlsSlot();
 
+// This is a compiler-generated struct that can be shared between hwasan
+// implementations.
+struct AccessInfo {
+  uptr addr;
+  uptr size;
+  bool is_store;
+  bool is_load;
+  bool recover;
+};
+
+// Given access info and frame information, unwind the stack and report the tag
+// mismatch.
+void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, void *uc,
+                       uptr *registers_frame = nullptr);
+
+// This dispatches to HandleTagMismatch but sets up the AccessInfo, program
+// counter, and frame pointer.
+void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame,
+                       size_t outsize);
+
 }  // namespace __hwasan
 
 #define HWASAN_MALLOC_HOOK(ptr, size)       \
@@ -162,4 +203,12 @@ typedef struct __hw_jmp_buf_struct __hw_jmp_buf[1];
 typedef struct __hw_jmp_buf_struct __hw_sigjmp_buf[1];
 #endif // HWASAN_WITH_INTERCEPTORS && __aarch64__
 
+#define ENSURE_HWASAN_INITED()      \
+  do {                              \
+    CHECK(!hwasan_init_is_running); \
+    if (!hwasan_inited) {           \
+      __hwasan_init();              \
+    }                               \
+  } while (0)
+
 #endif  // HWASAN_H
diff --git a/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp b/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp
new file mode 100644
index 000000000000..6c2a6077866f
--- /dev/null
+++ b/compiler-rt/lib/hwasan/hwasan_allocation_functions.cpp
@@ -0,0 +1,172 @@
+//===-- hwasan_allocation_functions.cpp -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of HWAddressSanitizer.
+//
+// Definitions for __sanitizer allocation functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "hwasan.h"
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_tls_get_addr.h"
+
+using namespace __hwasan;
+
+static uptr allocated_for_dlsym;
+static const uptr kDlsymAllocPoolSize = 1024;
+static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
+
+static bool IsInDlsymAllocPool(const void *ptr) {
+  uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+  return off < sizeof(alloc_memory_for_dlsym);
+}
+
+static void *AllocateFromLocalPool(uptr size_in_bytes) {
+  uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
+  void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
+  allocated_for_dlsym += size_in_words;
+  CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
+  return mem;
+}
+
+int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) {
+  GET_MALLOC_STACK_TRACE;
+  CHECK_NE(memptr, 0);
+  int res = hwasan_posix_memalign(memptr, alignment, size, &stack);
+  return res;
+}
+
+void *__sanitizer_memalign(uptr alignment, uptr size) {
+  GET_MALLOC_STACK_TRACE;
+  return hwasan_memalign(alignment, size, &stack);
+}
+
+void *__sanitizer_aligned_alloc(uptr alignment, uptr size) {
+  GET_MALLOC_STACK_TRACE;
+  return hwasan_aligned_alloc(alignment, size, &stack);
+}
+
+void *__sanitizer___libc_memalign(uptr alignment, uptr size) {
+  GET_MALLOC_STACK_TRACE;
+  void *ptr = hwasan_memalign(alignment, size, &stack);
+  if (ptr)
+    DTLS_on_libc_memalign(ptr, size);
+  return ptr;
+}
+
+void *__sanitizer_valloc(uptr size) {
+  GET_MALLOC_STACK_TRACE;
+  return hwasan_valloc(size, &stack);
+}
+
+void *__sanitizer_pvalloc(uptr size) {
+  GET_MALLOC_STACK_TRACE;
+  return hwasan_pvalloc(size, &stack);
+}
+
+void __sanitizer_free(void *ptr) {
+  GET_MALLOC_STACK_TRACE;
+  if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+    return;
+  hwasan_free(ptr, &stack);
+}
+
+void __sanitizer_cfree(void *ptr) {
+  GET_MALLOC_STACK_TRACE;
+  if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr)))
+    return;
+  hwasan_free(ptr, &stack);
+}
+
+uptr __sanitizer_malloc_usable_size(const void *ptr) {
+  return __sanitizer_get_allocated_size(ptr);
+}
+
+struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() {
+  __sanitizer_struct_mallinfo sret;
+  internal_memset(&sret, 0, sizeof(sret));
+  return sret;
+}
+
+int __sanitizer_mallopt(int cmd, int value) { return 0; }
+
+void __sanitizer_malloc_stats(void) {
+  // FIXME: implement, but don't call REAL(malloc_stats)!
+}
+
+void *__sanitizer_calloc(uptr nmemb, uptr size) {
+  GET_MALLOC_STACK_TRACE;
+  if (UNLIKELY(!hwasan_inited))
+    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
+    return AllocateFromLocalPool(nmemb * size);
+  return hwasan_calloc(nmemb, size, &stack);
+}
+
+void *__sanitizer_realloc(void *ptr, uptr size) {
+  GET_MALLOC_STACK_TRACE;
+  if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
+    uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
+    uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
+    void *new_ptr;
+    if (UNLIKELY(!hwasan_inited)) {
+      new_ptr = AllocateFromLocalPool(copy_size);
+    } else {
+      copy_size = size;
+      new_ptr = hwasan_malloc(copy_size, &stack);
+    }
+    internal_memcpy(new_ptr, ptr, copy_size);
+    return new_ptr;
+  }
+  return hwasan_realloc(ptr, size, &stack);
+}
+
+void *__sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) {
+  GET_MALLOC_STACK_TRACE;
+  return hwasan_reallocarray(ptr, nmemb, size, &stack);
+}
+
+void *__sanitizer_malloc(uptr size) {
+  GET_MALLOC_STACK_TRACE;
+  if (UNLIKELY(!hwasan_init_is_running))
+    ENSURE_HWASAN_INITED();
+  if (UNLIKELY(!hwasan_inited))
+    // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
+    return AllocateFromLocalPool(size);
+  return hwasan_malloc(size, &stack);
+}
+
+#if HWASAN_WITH_INTERCEPTORS
+#  define INTERCEPTOR_ALIAS(RET, FN, ARGS...)                                 \
+    extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS)               \
+        ALIAS("__sanitizer_" #FN);                                            \
+    extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN( \
+        ARGS) ALIAS("__sanitizer_" #FN)
+
+INTERCEPTOR_ALIAS(int, posix_memalign, void **memptr, SIZE_T alignment,
+                  SIZE_T size);
+INTERCEPTOR_ALIAS(void *, aligned_alloc, SIZE_T alignment, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, __libc_memalign, SIZE_T alignment, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, valloc, SIZE_T size);
+INTERCEPTOR_ALIAS(void, free, void *ptr);
+INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr);
+INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size);
+
+#  if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR_ALIAS(void *, memalign, SIZE_T alignment, SIZE_T size);
+INTERCEPTOR_ALIAS(void *, pvalloc, SIZE_T size);
+INTERCEPTOR_ALIAS(void, cfree, void *ptr);
+INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo);
+INTERCEPTOR_ALIAS(int, mallopt, int cmd, int value);
+INTERCEPTOR_ALIAS(void, malloc_stats, void);
+#  endif
+#endif  // #if HWASAN_WITH_INTERCEPTORS
diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.cpp b/compiler-rt/lib/hwasan/hwasan_allocator.cpp
index 0b6b7347892e..ef6d4d6c7678 100644
--- a/compiler-rt/lib/hwasan/hwasan_allocator.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_allocator.cpp
@@ -29,8 +29,8 @@ static AllocatorCache fallback_allocator_cache;
 static SpinMutex fallback_mutex;
 static atomic_uint8_t hwasan_allocator_tagging_enabled;
 
-static const tag_t kFallbackAllocTag = 0xBB;
-static const tag_t kFallbackFreeTag = 0xBC;
+static constexpr tag_t kFallbackAllocTag = 0xBB & kTagMask;
+static constexpr tag_t kFallbackFreeTag = 0xBC;
 
 enum RightAlignMode {
   kRightAlignNever,
@@ -80,11 +80,29 @@ void GetAllocatorStats(AllocatorStatCounters s) {
   allocator.GetStats(s);
 }
 
+uptr GetAliasRegionStart() {
+#if defined(HWASAN_ALIASING_MODE)
+  constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1);
+  uptr AliasRegionStart =
+      __hwasan_shadow_memory_dynamic_address + kAliasRegionOffset;
+
+  CHECK_EQ(AliasRegionStart >> kTaggableRegionCheckShift,
+           __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
+  CHECK_EQ(
+      (AliasRegionStart + kAliasRegionOffset - 1) >> kTaggableRegionCheckShift,
+      __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
+  return AliasRegionStart;
+#else
+  return 0;
+#endif
+}
+
 void HwasanAllocatorInit() {
   atomic_store_relaxed(&hwasan_allocator_tagging_enabled,
                        !flags()->disable_allocator_tagging);
   SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
-  allocator.Init(common_flags()->allocator_release_to_os_interval_ms);
+  allocator.Init(common_flags()->allocator_release_to_os_interval_ms,
+                 GetAliasRegionStart());
   for (uptr i = 0; i < sizeof(tail_magic); i++)
     tail_magic[i] = GetCurrentThread()->GenerateRandomTag();
 }
@@ -148,7 +166,8 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment,
   // Tagging can only be skipped when both tag_in_malloc and tag_in_free are
   // false. When tag_in_malloc = false and tag_in_free = true malloc needs to
   // retag to 0.
-  if ((flags()->tag_in_malloc || flags()->tag_in_free) &&
+  if (InTaggableRegion(reinterpret_cast<uptr>(user_ptr)) &&
+      (flags()->tag_in_malloc || flags()->tag_in_free) &&
       atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) {
     if (flags()->tag_in_malloc && malloc_bisect(stack, orig_size)) {
       tag_t tag = t ? t->GenerateRandomTag() : kFallbackAllocTag;
@@ -175,6 +194,8 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment,
 static bool PointerAndMemoryTagsMatch(void *tagged_ptr) {
   CHECK(tagged_ptr);
   uptr tagged_uptr = reinterpret_cast<uptr>(tagged_ptr);
+  if (!InTaggableRegion(tagged_uptr))
+    return true;
   tag_t mem_tag = *reinterpret_cast<tag_t *>(
       MemToShadow(reinterpret_cast<uptr>(UntagPtr(tagged_ptr))));
   return PossiblyShortTagMatches(mem_tag, tagged_uptr, 1);
@@ -187,9 +208,12 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
   if (!PointerAndMemoryTagsMatch(tagged_ptr))
     ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr));
 
-  void *untagged_ptr = UntagPtr(tagged_ptr);
+  void *untagged_ptr = InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr))
+                           ? UntagPtr(tagged_ptr)
+                           : tagged_ptr;
   void *aligned_ptr = reinterpret_cast<void *>(
       RoundDownTo(reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment));
+  tag_t pointer_tag = GetTagFromPointer(reinterpret_cast<uptr>(tagged_ptr));
   Metadata *meta =
       reinterpret_cast<Metadata *>(allocator.GetMetaData(aligned_ptr));
   uptr orig_size = meta->get_requested_size();
@@ -219,10 +243,27 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
         Min(TaggedSize(orig_size), (uptr)flags()->max_free_fill_size);
     internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size);
   }
-  if (flags()->tag_in_free && malloc_bisect(stack, 0) &&
-      atomic_load_relaxed(&hwasan_allocator_tagging_enabled))
+  if (InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr)) &&
+      flags()->tag_in_free && malloc_bisect(stack, 0) &&
+      atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) {
+    // Always store full 8-bit tags on free to maximize UAF detection.
+    tag_t tag;
+    if (t) {
+      // Make sure we are not using a short granule tag as a poison tag. This
+      // would make us attempt to read the memory on a UaF.
+      // The tag can be zero if tagging is disabled on this thread.
+      do {
+        tag = t->GenerateRandomTag(/*num_bits=*/8);
+      } while (
+          UNLIKELY((tag < kShadowAlignment || tag == pointer_tag) && tag != 0));
+    } else {
+      static_assert(kFallbackFreeTag >= kShadowAlignment,
+                    "fallback tag must not be a short granule tag.");
+      tag = kFallbackFreeTag;
+    }
     TagMemoryAligned(reinterpret_cast<uptr>(aligned_ptr), TaggedSize(orig_size),
-                     t ? t->GenerateRandomTag() : kFallbackFreeTag);
+                     tag);
+  }
   if (t) {
     allocator.Deallocate(t->allocator_cache(), aligned_ptr);
     if (auto *ha = t->heap_allocations())
@@ -365,7 +406,7 @@ int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
     // OOM error is already taken care of by HwasanAllocate.
     return errno_ENOMEM;
   CHECK(IsAligned((uptr)ptr, alignment));
-  *(void **)UntagPtr(memptr) = ptr;
+  *memptr = ptr;
   return 0;
 }
 
diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.h b/compiler-rt/lib/hwasan/hwasan_allocator.h
index 43670a6a3fb7..35c3d6b4bf43 100644
--- a/compiler-rt/lib/hwasan/hwasan_allocator.h
+++ b/compiler-rt/lib/hwasan/hwasan_allocator.h
@@ -13,13 +13,16 @@
 #ifndef HWASAN_ALLOCATOR_H
 #define HWASAN_ALLOCATOR_H
 
+#include "hwasan.h"
+#include "hwasan_interface_internal.h"
+#include "hwasan_mapping.h"
+#include "hwasan_poisoning.h"
 #include "sanitizer_common/sanitizer_allocator.h"
 #include "sanitizer_common/sanitizer_allocator_checks.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
 #include "sanitizer_common/sanitizer_allocator_report.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_ring_buffer.h"
-#include "hwasan_poisoning.h"
 
 #if !defined(__aarch64__) && !defined(__x86_64__)
 #error Unsupported platform
@@ -55,7 +58,12 @@ static const uptr kMaxAllowedMallocSize = 1UL << 40;  // 1T
 
 struct AP64 {
   static const uptr kSpaceBeg = ~0ULL;
+
+#if defined(HWASAN_ALIASING_MODE)
+  static const uptr kSpaceSize = 1ULL << kAddressTagShift;
+#else
   static const uptr kSpaceSize = 0x2000000000ULL;
+#endif
   static const uptr kMetadataSize = sizeof(Metadata);
   typedef __sanitizer::VeryDenseSizeClassMap SizeClassMap;
   using AddressSpaceView = LocalAddressSpaceView;
@@ -102,6 +110,16 @@ typedef RingBuffer<HeapAllocationRecord> HeapAllocationsRingBuffer;
 
 void GetAllocatorStats(AllocatorStatCounters s);
 
+inline bool InTaggableRegion(uptr addr) {
+#if defined(HWASAN_ALIASING_MODE)
+  // Aliases are mapped next to shadow so that the upper bits match the shadow
+  // base.
+  return (addr >> kTaggableRegionCheckShift) ==
+         (GetShadowOffset() >> kTaggableRegionCheckShift);
+#endif
+  return true;
+}
+
 } // namespace __hwasan
 
 #endif // HWASAN_ALLOCATOR_H
diff --git a/compiler-rt/lib/hwasan/hwasan_blacklist.txt b/compiler-rt/lib/hwasan/hwasan_blacklist.txt
deleted file mode 100644
index 395ba28f0212..000000000000
--- a/compiler-rt/lib/hwasan/hwasan_blacklist.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-# Blacklist for HWAddressSanitizer. Turns off instrumentation of particular
-# functions or sources. Use with care. You may set location of blacklist
-# at compile-time using -fsanitize-blacklist=<path> flag.
-
-# Example usage:
-# fun:*bad_function_name*
-# src:file_with_tricky_code.cc
diff --git a/compiler-rt/lib/hwasan/hwasan_checks.h b/compiler-rt/lib/hwasan/hwasan_checks.h
index a8de0fef20f0..ab543ea88beb 100644
--- a/compiler-rt/lib/hwasan/hwasan_checks.h
+++ b/compiler-rt/lib/hwasan/hwasan_checks.h
@@ -13,6 +13,7 @@
 #ifndef HWASAN_CHECKS_H
 #define HWASAN_CHECKS_H
 
+#include "hwasan_allocator.h"
 #include "hwasan_mapping.h"
 #include "sanitizer_common/sanitizer_common.h"
 
@@ -81,6 +82,8 @@ enum class AccessType { Load, Store };
 
 template <ErrorAction EA, AccessType AT, unsigned LogSize>
 __attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) {
+  if (!InTaggableRegion(p))
+    return;
   uptr ptr_raw = p & ~kAddressTagMask;
   tag_t mem_tag = *(tag_t *)MemToShadow(ptr_raw);
   if (UNLIKELY(!PossiblyShortTagMatches(mem_tag, p, 1 << LogSize))) {
@@ -94,7 +97,7 @@ __attribute__((always_inline, nodebug)) static void CheckAddress(uptr p) {
 template <ErrorAction EA, AccessType AT>
 __attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p,
                                                                       uptr sz) {
-  if (sz == 0)
+  if (sz == 0 || !InTaggableRegion(p))
     return;
   tag_t ptr_tag = GetTagFromPointer(p);
   uptr ptr_raw = p & ~kAddressTagMask;
diff --git a/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp b/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp
index 12730b29bae3..7642ba6c0bf0 100644
--- a/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_dynamic_shadow.cpp
@@ -12,15 +12,17 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "hwasan.h"
 #include "hwasan_dynamic_shadow.h"
-#include "hwasan_mapping.h"
-#include "sanitizer_common/sanitizer_common.h"
-#include "sanitizer_common/sanitizer_posix.h"
 
 #include <elf.h>
 #include <link.h>
 
+#include "hwasan.h"
+#include "hwasan_mapping.h"
+#include "hwasan_thread_list.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_posix.h"
+
 // The code in this file needs to run in an unrelocated binary. It should not
 // access any external symbol, including its own non-hidden globals.
 
@@ -111,12 +113,27 @@ uptr FindDynamicShadowStart(uptr shadow_size_bytes) {
 }
 
 }  // namespace __hwasan
+
+#elif SANITIZER_FUCHSIA
+
+namespace __hwasan {
+
+void InitShadowGOT() {}
+
+}  // namespace __hwasan
+
 #else
 namespace __hwasan {
 
 void InitShadowGOT() {}
 
 uptr FindDynamicShadowStart(uptr shadow_size_bytes) {
+#  if defined(HWASAN_ALIASING_MODE)
+  constexpr uptr kAliasSize = 1ULL << kAddressTagShift;
+  constexpr uptr kNumAliases = 1ULL << kTagBits;
+  return MapDynamicShadowAndAliases(shadow_size_bytes, kAliasSize, kNumAliases,
+                                    RingBufferSize());
+#  endif
   return MapDynamicShadow(shadow_size_bytes, kShadowScale, kShadowBaseAlignment,
                           kHighMemEnd);
 }
diff --git a/compiler-rt/lib/hwasan/hwasan_flags.h b/compiler-rt/lib/hwasan/hwasan_flags.h
index 0a6998f675d6..b17750158d02 100644
--- a/compiler-rt/lib/hwasan/hwasan_flags.h
+++ b/compiler-rt/lib/hwasan/hwasan_flags.h
@@ -12,6 +12,8 @@
 #ifndef HWASAN_FLAGS_H
 #define HWASAN_FLAGS_H
 
+#include "sanitizer_common/sanitizer_internal_defs.h"
+
 namespace __hwasan {
 
 struct Flags {
diff --git a/compiler-rt/lib/hwasan/hwasan_flags.inc b/compiler-rt/lib/hwasan/hwasan_flags.inc
index 8e431d9c4ff9..18ea47f981be 100644
--- a/compiler-rt/lib/hwasan/hwasan_flags.inc
+++ b/compiler-rt/lib/hwasan/hwasan_flags.inc
@@ -72,3 +72,12 @@ HWASAN_FLAG(uptr, malloc_bisect_right, 0,
 HWASAN_FLAG(bool, malloc_bisect_dump, false,
             "Print all allocations within [malloc_bisect_left, "
             "malloc_bisect_right] range ")
+
+
+// Exit if we fail to enable the AArch64 kernel ABI relaxation which allows
+// tagged pointers in syscalls.  This is the default, but being able to disable
+// that behaviour is useful for running the testsuite on more platforms (the
+// testsuite can run since we manually ensure any pointer arguments to syscalls
+// are untagged before the call.
+HWASAN_FLAG(bool, fail_without_syscall_abi, true,
+            "Exit if fail to request relaxed syscall ABI.")
diff --git a/compiler-rt/lib/hwasan/hwasan_fuchsia.cpp b/compiler-rt/lib/hwasan/hwasan_fuchsia.cpp
new file mode 100644
index 000000000000..e299a7e862eb
--- /dev/null
+++ b/compiler-rt/lib/hwasan/hwasan_fuchsia.cpp
@@ -0,0 +1,213 @@
+//===-- hwasan_fuchsia.cpp --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file is a part of HWAddressSanitizer and contains Fuchsia-specific
+/// code.
+///
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_fuchsia.h"
+#if SANITIZER_FUCHSIA
+
+#include "hwasan.h"
+#include "hwasan_interface_internal.h"
+#include "hwasan_report.h"
+#include "hwasan_thread.h"
+#include "hwasan_thread_list.h"
+
+// This TLS variable contains the location of the stack ring buffer and can be
+// used to always find the hwasan thread object associated with the current
+// running thread.
+[[gnu::tls_model("initial-exec")]]
+SANITIZER_INTERFACE_ATTRIBUTE
+THREADLOCAL uptr __hwasan_tls;
+
+namespace __hwasan {
+
+bool InitShadow() {
+  __sanitizer::InitShadowBounds();
+  CHECK_NE(__sanitizer::ShadowBounds.shadow_limit, 0);
+
+  // These variables are used by MemIsShadow for asserting we have a correct
+  // shadow address. On Fuchsia, we only have one region of shadow, so the
+  // bounds of Low shadow can be zero while High shadow represents the true
+  // bounds. Note that these are inclusive ranges.
+  kLowShadowStart = 0;
+  kLowShadowEnd = 0;
+  kHighShadowStart = __sanitizer::ShadowBounds.shadow_base;
+  kHighShadowEnd = __sanitizer::ShadowBounds.shadow_limit - 1;
+
+  return true;
+}
+
+bool MemIsApp(uptr p) {
+  CHECK(GetTagFromPointer(p) == 0);
+  return __sanitizer::ShadowBounds.shadow_limit <= p &&
+         p <= (__sanitizer::ShadowBounds.memory_limit - 1);
+}
+
+// These are known parameters passed to the hwasan runtime on thread creation.
+struct Thread::InitState {
+  uptr stack_bottom, stack_top;
+};
+
+static void FinishThreadInitialization(Thread *thread);
+
+void InitThreads() {
+  // This is the minimal alignment needed for the storage where hwasan threads
+  // and their stack ring buffers are placed. This alignment is necessary so the
+  // stack ring buffer can perform a simple calculation to get the next element
+  // in the RB. The instructions for this calculation are emitted by the
+  // compiler. (Full explanation in hwasan_thread_list.h.)
+  uptr alloc_size = UINT64_C(1) << kShadowBaseAlignment;
+  uptr thread_start = reinterpret_cast<uptr>(
+      MmapAlignedOrDieOnFatalError(alloc_size, alloc_size, __func__));
+
+  InitThreadList(thread_start, alloc_size);
+
+  // Create the hwasan thread object for the current (main) thread. Stack info
+  // for this thread is known from information passed via
+  // __sanitizer_startup_hook.
+  const Thread::InitState state = {
+      .stack_bottom = __sanitizer::MainThreadStackBase,
+      .stack_top =
+          __sanitizer::MainThreadStackBase + __sanitizer::MainThreadStackSize,
+  };
+  FinishThreadInitialization(hwasanThreadList().CreateCurrentThread(&state));
+}
+
+uptr *GetCurrentThreadLongPtr() { return &__hwasan_tls; }
+
+// This is called from the parent thread before the new thread is created. Here
+// we can propagate known info like the stack bounds to Thread::Init before
+// jumping into the thread. We cannot initialize the stack ring buffer yet since
+// we have not entered the new thread.
+static void *BeforeThreadCreateHook(uptr user_id, bool detached,
+                                    const char *name, uptr stack_bottom,
+                                    uptr stack_size) {
+  const Thread::InitState state = {
+      .stack_bottom = stack_bottom,
+      .stack_top = stack_bottom + stack_size,
+  };
+  return hwasanThreadList().CreateCurrentThread(&state);
+}
+
+// This sets the stack top and bottom according to the InitState passed to
+// CreateCurrentThread above.
+void Thread::InitStackAndTls(const InitState *state) {
+  CHECK_NE(state->stack_bottom, 0);
+  CHECK_NE(state->stack_top, 0);
+  stack_bottom_ = state->stack_bottom;
+  stack_top_ = state->stack_top;
+  tls_end_ = tls_begin_ = 0;
+}
+
+// This is called after creating a new thread with the pointer returned by
+// BeforeThreadCreateHook. We are still in the creating thread and should check
+// if it was actually created correctly.
+static void ThreadCreateHook(void *hook, bool aborted) {
+  Thread *thread = static_cast<Thread *>(hook);
+  if (!aborted) {
+    // The thread was created successfully.
+    // ThreadStartHook can already be running in the new thread.
+  } else {
+    // The thread wasn't created after all.
+    // Clean up everything we set up in BeforeThreadCreateHook.
+    atomic_signal_fence(memory_order_seq_cst);
+    hwasanThreadList().ReleaseThread(thread);
+  }
+}
+
+// This is called in the newly-created thread before it runs anything else,
+// with the pointer returned by BeforeThreadCreateHook (above). Here we can
+// setup the stack ring buffer.
+static void ThreadStartHook(void *hook, thrd_t self) {
+  Thread *thread = static_cast<Thread *>(hook);
+  FinishThreadInitialization(thread);
+  thread->InitRandomState();
+}
+
+// This is the function that sets up the stack ring buffer and enables us to use
+// GetCurrentThread. This function should only be called while IN the thread
+// that we want to create the hwasan thread object for so __hwasan_tls can be
+// properly referenced.
+static void FinishThreadInitialization(Thread *thread) {
+  CHECK_NE(thread, nullptr);
+
+  // The ring buffer is located immediately before the thread object.
+  uptr stack_buffer_size = hwasanThreadList().GetRingBufferSize();
+  uptr stack_buffer_start = reinterpret_cast<uptr>(thread) - stack_buffer_size;
+  thread->InitStackRingBuffer(stack_buffer_start, stack_buffer_size);
+}
+
+static void ThreadExitHook(void *hook, thrd_t self) {
+  Thread *thread = static_cast<Thread *>(hook);
+  atomic_signal_fence(memory_order_seq_cst);
+  hwasanThreadList().ReleaseThread(thread);
+}
+
+uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
+  CHECK(IsAligned(p, kShadowAlignment));
+  CHECK(IsAligned(size, kShadowAlignment));
+  __sanitizer_fill_shadow(p, size, tag,
+                          common_flags()->clear_shadow_mmap_threshold);
+  return AddTagToPointer(p, tag);
+}
+
+// Not implemented because Fuchsia does not use signal handlers.
+void HwasanOnDeadlySignal(int signo, void *info, void *context) {}
+
+// Not implemented because Fuchsia does not use interceptors.
+void InitializeInterceptors() {}
+
+// Not implemented because this is only relevant for Android.
+void AndroidTestTlsSlot() {}
+
+// TSD was normally used on linux as a means of calling the hwasan thread exit
+// handler passed to pthread_key_create. This is not needed on Fuchsia because
+// we will be using __sanitizer_thread_exit_hook.
+void HwasanTSDInit() {}
+void HwasanTSDThreadInit() {}
+
+// On linux, this just would call `atexit(HwasanAtExit)`. The functions in
+// HwasanAtExit are unimplemented for Fuchsia and effectively no-ops, so this
+// function is unneeded.
+void InstallAtExitHandler() {}
+
+// TODO(fxbug.dev/81499): Once we finalize the tagged pointer ABI in zircon, we should come back
+// here and implement the appropriate check that TBI is enabled.
+void InitializeOsSupport() {}
+
+}  // namespace __hwasan
+
+extern "C" {
+
+void *__sanitizer_before_thread_create_hook(thrd_t thread, bool detached,
+                                            const char *name, void *stack_base,
+                                            size_t stack_size) {
+  return __hwasan::BeforeThreadCreateHook(
+      reinterpret_cast<uptr>(thread), detached, name,
+      reinterpret_cast<uptr>(stack_base), stack_size);
+}
+
+void __sanitizer_thread_create_hook(void *hook, thrd_t thread, int error) {
+  __hwasan::ThreadCreateHook(hook, error != thrd_success);
+}
+
+void __sanitizer_thread_start_hook(void *hook, thrd_t self) {
+  __hwasan::ThreadStartHook(hook, reinterpret_cast<uptr>(self));
+}
+
+void __sanitizer_thread_exit_hook(void *hook, thrd_t self) {
+  __hwasan::ThreadExitHook(hook, self);
+}
+
+}  // extern "C"
+
+#endif  // SANITIZER_FUCHSIA
diff --git a/compiler-rt/lib/hwasan/hwasan_ignorelist.txt b/compiler-rt/lib/hwasan/hwasan_ignorelist.txt
new file mode 100644
index 000000000000..70590c970f55
--- /dev/null
+++ b/compiler-rt/lib/hwasan/hwasan_ignorelist.txt
@@ -0,0 +1,7 @@
+# Ignorelist for HWAddressSanitizer. Turns off instrumentation of particular
+# functions or sources. Use with care. You may set location of ignorelist
+# at compile-time using -fsanitize-ignorelist=<path> flag.
+
+# Example usage:
+# fun:*bad_function_name*
+# src:file_with_tricky_code.cc
diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
index 44e569ee6d72..68f8adec0776 100644
--- a/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_interceptors.cpp
@@ -16,192 +16,14 @@
 
 #include "interception/interception.h"
 #include "hwasan.h"
-#include "hwasan_allocator.h"
-#include "hwasan_mapping.h"
 #include "hwasan_thread.h"
-#include "hwasan_poisoning.h"
-#include "hwasan_report.h"
-#include "sanitizer_common/sanitizer_platform_limits_posix.h"
-#include "sanitizer_common/sanitizer_allocator.h"
-#include "sanitizer_common/sanitizer_allocator_interface.h"
-#include "sanitizer_common/sanitizer_allocator_internal.h"
-#include "sanitizer_common/sanitizer_atomic.h"
-#include "sanitizer_common/sanitizer_common.h"
-#include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
-#include "sanitizer_common/sanitizer_libc.h"
-#include "sanitizer_common/sanitizer_linux.h"
-#include "sanitizer_common/sanitizer_tls_get_addr.h"
 
-#include <stdarg.h>
-// ACHTUNG! No other system header includes in this file.
-// Ideally, we should get rid of stdarg.h as well.
+#if !SANITIZER_FUCHSIA
 
 using namespace __hwasan;
 
-using __sanitizer::memory_order;
-using __sanitizer::atomic_load;
-using __sanitizer::atomic_store;
-using __sanitizer::atomic_uintptr_t;
-
-static uptr allocated_for_dlsym;
-static const uptr kDlsymAllocPoolSize = 1024;
-static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize];
-
-static bool IsInDlsymAllocPool(const void *ptr) {
-  uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-  return off < sizeof(alloc_memory_for_dlsym);
-}
-
-static void *AllocateFromLocalPool(uptr size_in_bytes) {
-  uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize;
-  void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym];
-  allocated_for_dlsym += size_in_words;
-  CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize);
-  return mem;
-}
-
-#define ENSURE_HWASAN_INITED() do { \
-  CHECK(!hwasan_init_is_running); \
-  if (!hwasan_inited) { \
-    __hwasan_init(); \
-  } \
-} while (0)
-
-
-int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) {
-  GET_MALLOC_STACK_TRACE;
-  CHECK_NE(memptr, 0);
-  int res = hwasan_posix_memalign(memptr, alignment, size, &stack);
-  return res;
-}
-
-void * __sanitizer_memalign(uptr alignment, uptr size) {
-  GET_MALLOC_STACK_TRACE;
-  return hwasan_memalign(alignment, size, &stack);
-}
-
-void * __sanitizer_aligned_alloc(uptr alignment, uptr size) {
-  GET_MALLOC_STACK_TRACE;
-  return hwasan_aligned_alloc(alignment, size, &stack);
-}
-
-void * __sanitizer___libc_memalign(uptr alignment, uptr size) {
-  GET_MALLOC_STACK_TRACE;
-  void *ptr = hwasan_memalign(alignment, size, &stack);
-  if (ptr)
-    DTLS_on_libc_memalign(ptr, size);
-  return ptr;
-}
-
-void * __sanitizer_valloc(uptr size) {
-  GET_MALLOC_STACK_TRACE;
-  return hwasan_valloc(size, &stack);
-}
-
-void * __sanitizer_pvalloc(uptr size) {
-  GET_MALLOC_STACK_TRACE;
-  return hwasan_pvalloc(size, &stack);
-}
-
-void __sanitizer_free(void *ptr) {
-  GET_MALLOC_STACK_TRACE;
-  if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
-  hwasan_free(ptr, &stack);
-}
-
-void __sanitizer_cfree(void *ptr) {
-  GET_MALLOC_STACK_TRACE;
-  if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return;
-  hwasan_free(ptr, &stack);
-}
-
-uptr __sanitizer_malloc_usable_size(const void *ptr) {
-  return __sanitizer_get_allocated_size(ptr);
-}
-
-struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() {
-  __sanitizer_struct_mallinfo sret;
-  internal_memset(&sret, 0, sizeof(sret));
-  return sret;
-}
-
-int __sanitizer_mallopt(int cmd, int value) {
-  return 0;
-}
-
-void __sanitizer_malloc_stats(void) {
-  // FIXME: implement, but don't call REAL(malloc_stats)!
-}
-
-void * __sanitizer_calloc(uptr nmemb, uptr size) {
-  GET_MALLOC_STACK_TRACE;
-  if (UNLIKELY(!hwasan_inited))
-    // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(nmemb * size);
-  return hwasan_calloc(nmemb, size, &stack);
-}
-
-void * __sanitizer_realloc(void *ptr, uptr size) {
-  GET_MALLOC_STACK_TRACE;
-  if (UNLIKELY(IsInDlsymAllocPool(ptr))) {
-    uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym;
-    uptr copy_size = Min(size, kDlsymAllocPoolSize - offset);
-    void *new_ptr;
-    if (UNLIKELY(!hwasan_inited)) {
-      new_ptr = AllocateFromLocalPool(copy_size);
-    } else {
-      copy_size = size;
-      new_ptr = hwasan_malloc(copy_size, &stack);
-    }
-    internal_memcpy(new_ptr, ptr, copy_size);
-    return new_ptr;
-  }
-  return hwasan_realloc(ptr, size, &stack);
-}
-
-void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) {
-  GET_MALLOC_STACK_TRACE;
-  return hwasan_reallocarray(ptr, nmemb, size, &stack);
-}
-
-void * __sanitizer_malloc(uptr size) {
-  GET_MALLOC_STACK_TRACE;
-  if (UNLIKELY(!hwasan_init_is_running))
-    ENSURE_HWASAN_INITED();
-  if (UNLIKELY(!hwasan_inited))
-    // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym.
-    return AllocateFromLocalPool(size);
-  return hwasan_malloc(size, &stack);
-}
-
 #if HWASAN_WITH_INTERCEPTORS
-#define INTERCEPTOR_ALIAS(RET, FN, ARGS...)                                  \
-  extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS)                \
-      ALIAS("__sanitizer_" #FN);                                             \
-  extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN(  \
-      ARGS) ALIAS("__sanitizer_" #FN)
-
-INTERCEPTOR_ALIAS(int, posix_memalign, void **memptr, SIZE_T alignment,
-                  SIZE_T size);
-INTERCEPTOR_ALIAS(void *, aligned_alloc, SIZE_T alignment, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, __libc_memalign, SIZE_T alignment, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, valloc, SIZE_T size);
-INTERCEPTOR_ALIAS(void, free, void *ptr);
-INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr);
-INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size);
-
-#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
-INTERCEPTOR_ALIAS(void *, memalign, SIZE_T alignment, SIZE_T size);
-INTERCEPTOR_ALIAS(void *, pvalloc, SIZE_T size);
-INTERCEPTOR_ALIAS(void, cfree, void *ptr);
-INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo);
-INTERCEPTOR_ALIAS(int, mallopt, int cmd, int value);
-INTERCEPTOR_ALIAS(void, malloc_stats, void);
-#endif
 
 struct ThreadStartArg {
   thread_callback_t callback;
@@ -221,8 +43,7 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*),
   ThreadStartArg *A = reinterpret_cast<ThreadStartArg *> (MmapOrDie(
       GetPageSizeCached(), "pthread_create"));
   *A = {callback, param};
-  int res = REAL(pthread_create)(UntagPtr(th), UntagPtr(attr),
-                                 &HwasanThreadStartFunc, A);
+  int res = REAL(pthread_create)(th, attr, &HwasanThreadStartFunc, A);
   return res;
 }
 
@@ -347,3 +168,5 @@ void InitializeInterceptors() {
   inited = 1;
 }
 } // namespace __hwasan
+
+#endif  // #if !SANITIZER_FUCHSIA
diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S b/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S
index 23d565936d87..fd20825e3dac 100644
--- a/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S
+++ b/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S
@@ -1,4 +1,5 @@
 #include "sanitizer_common/sanitizer_asm.h"
+#include "builtins/assembly.h"
 
 #if defined(__linux__) && HWASAN_WITH_INTERCEPTORS
 #define COMMON_INTERCEPTOR_SPILL_AREA __hwasan_extra_spill_area
@@ -9,3 +10,5 @@
 #endif
 
 NO_EXEC_STACK_DIRECTIVE
+
+GNU_PROPERTY_BTI_PAC
diff --git a/compiler-rt/lib/hwasan/hwasan_linux.cpp b/compiler-rt/lib/hwasan/hwasan_linux.cpp
index e99926d355cf..e22723529f44 100644
--- a/compiler-rt/lib/hwasan/hwasan_linux.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_linux.cpp
@@ -69,10 +69,6 @@ static void ProtectGap(uptr addr, uptr size) {
 
 uptr kLowMemStart;
 uptr kLowMemEnd;
-uptr kLowShadowEnd;
-uptr kLowShadowStart;
-uptr kHighShadowStart;
-uptr kHighShadowEnd;
 uptr kHighMemStart;
 uptr kHighMemEnd;
 
@@ -114,37 +110,59 @@ static void InitializeShadowBaseAddress(uptr shadow_size_bytes) {
       FindDynamicShadowStart(shadow_size_bytes);
 }
 
-void InitPrctl() {
+void InitializeOsSupport() {
 #define PR_SET_TAGGED_ADDR_CTRL 55
 #define PR_GET_TAGGED_ADDR_CTRL 56
 #define PR_TAGGED_ADDR_ENABLE (1UL << 0)
   // Check we're running on a kernel that can use the tagged address ABI.
-  if (internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0) == (uptr)-1 &&
-      errno == EINVAL) {
-#if SANITIZER_ANDROID
+  int local_errno = 0;
+  if (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0),
+                       &local_errno) &&
+      local_errno == EINVAL) {
+#  if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE)
     // Some older Android kernels have the tagged pointer ABI on
     // unconditionally, and hence don't have the tagged-addr prctl while still
     // allow the ABI.
     // If targeting Android and the prctl is not around we assume this is the
     // case.
     return;
-#else
-    Printf(
-        "FATAL: "
-        "HWAddressSanitizer requires a kernel with tagged address ABI.\n");
-    Die();
-#endif
+#  else
+    if (flags()->fail_without_syscall_abi) {
+      Printf(
+          "FATAL: "
+          "HWAddressSanitizer requires a kernel with tagged address ABI.\n");
+      Die();
+    }
+#  endif
   }
 
   // Turn on the tagged address ABI.
-  if (internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, 0, 0, 0) ==
-          (uptr)-1 ||
-      !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) {
-    Printf(
-        "FATAL: HWAddressSanitizer failed to enable tagged address syscall "
-        "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` "
-        "configuration.\n");
-    Die();
+  if ((internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL,
+                                       PR_TAGGED_ADDR_ENABLE, 0, 0, 0)) ||
+       !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0))) {
+#  if defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
+    // Try the new prctl API for Intel LAM.  The API is based on a currently
+    // unsubmitted patch to the Linux kernel (as of May 2021) and is thus
+    // subject to change.  Patch is here:
+    // https://lore.kernel.org/linux-mm/20210205151631.43511-12-kirill.shutemov@linux.intel.com/
+    int tag_bits = kTagBits;
+    int tag_shift = kAddressTagShift;
+    if (!internal_iserror(
+            internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE,
+                           reinterpret_cast<unsigned long>(&tag_bits),
+                           reinterpret_cast<unsigned long>(&tag_shift), 0))) {
+      CHECK_EQ(tag_bits, kTagBits);
+      CHECK_EQ(tag_shift, kAddressTagShift);
+      return;
+    }
+#  endif  // defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE)
+    if (flags()->fail_without_syscall_abi) {
+      Printf(
+          "FATAL: HWAddressSanitizer failed to enable tagged address syscall "
+          "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` "
+          "configuration.\n");
+      Die();
+    }
   }
 #undef PR_SET_TAGGED_ADDR_CTRL
 #undef PR_GET_TAGGED_ADDR_CTRL
@@ -214,23 +232,16 @@ void InitThreads() {
   ProtectGap(thread_space_end,
              __hwasan_shadow_memory_dynamic_address - thread_space_end);
   InitThreadList(thread_space_start, thread_space_end - thread_space_start);
+  hwasanThreadList().CreateCurrentThread();
 }
 
 bool MemIsApp(uptr p) {
+// Memory outside the alias range has non-zero tags.
+#  if !defined(HWASAN_ALIASING_MODE)
   CHECK(GetTagFromPointer(p) == 0);
-  return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
-}
+#  endif
 
-static void HwasanAtExit(void) {
-  if (common_flags()->print_module_map)
-    DumpProcessMap();
-  if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0))
-    ReportStats();
-  if (hwasan_report_count > 0) {
-    // ReportAtExitStatistics();
-    if (common_flags()->exitcode)
-      internal__exit(common_flags()->exitcode);
-  }
+  return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd);
 }
 
 void InstallAtExitHandler() {
@@ -309,22 +320,6 @@ void AndroidTestTlsSlot() {
 void AndroidTestTlsSlot() {}
 #endif
 
-Thread *GetCurrentThread() {
-  uptr *ThreadLongPtr = GetCurrentThreadLongPtr();
-  if (UNLIKELY(*ThreadLongPtr == 0))
-    return nullptr;
-  auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr;
-  return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next());
-}
-
-struct AccessInfo {
-  uptr addr;
-  uptr size;
-  bool is_store;
-  bool is_load;
-  bool recover;
-};
-
 static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
   // Access type is passed in a platform dependent way (see below) and encoded
   // as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is
@@ -375,28 +370,6 @@ static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
   return AccessInfo{addr, size, is_store, !is_store, recover};
 }
 
-static void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame,
-                              ucontext_t *uc, uptr *registers_frame = nullptr) {
-  InternalMmapVector<BufferedStackTrace> stack_buffer(1);
-  BufferedStackTrace *stack = stack_buffer.data();
-  stack->Reset();
-  stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal);
-
-  // The second stack frame contains the failure __hwasan_check function, as
-  // we have a stack frame for the registers saved in __hwasan_tag_mismatch that
-  // we wish to ignore. This (currently) only occurs on AArch64, as x64
-  // implementations use SIGTRAP to implement the failure, and thus do not go
-  // through the stack saver.
-  if (registers_frame && stack->trace && stack->size > 0) {
-    stack->trace++;
-    stack->size--;
-  }
-
-  bool fatal = flags()->halt_on_error || !ai.recover;
-  ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal,
-                    registers_frame);
-}
-
 static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) {
   AccessInfo ai = GetAccessInfo(info, uc);
   if (!ai.is_store && !ai.is_load)
@@ -429,27 +402,39 @@ void HwasanOnDeadlySignal(int signo, void *info, void *context) {
   HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
 }
 
+void Thread::InitStackAndTls(const InitState *) {
+  uptr tls_size;
+  uptr stack_size;
+  GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_,
+                       &tls_size);
+  stack_top_ = stack_bottom_ + stack_size;
+  tls_end_ = tls_begin_ + tls_size;
+}
 
-} // namespace __hwasan
-
-// Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the
-// rest of the mismatch handling code (C++).
-void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame,
-                            size_t outsize) {
-  __hwasan::AccessInfo ai;
-  ai.is_store = access_info & 0x10;
-  ai.is_load = !ai.is_store;
-  ai.recover = access_info & 0x20;
-  ai.addr = addr;
-  if ((access_info & 0xf) == 0xf)
-    ai.size = outsize;
-  else
-    ai.size = 1 << (access_info & 0xf);
-
-  __hwasan::HandleTagMismatch(ai, (uptr)__builtin_return_address(0),
-                              (uptr)__builtin_frame_address(0), nullptr,
-                              registers_frame);
-  __builtin_unreachable();
+uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
+  CHECK(IsAligned(p, kShadowAlignment));
+  CHECK(IsAligned(size, kShadowAlignment));
+  uptr shadow_start = MemToShadow(p);
+  uptr shadow_size = MemToShadowSize(size);
+
+  uptr page_size = GetPageSizeCached();
+  uptr page_start = RoundUpTo(shadow_start, page_size);
+  uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size);
+  uptr threshold = common_flags()->clear_shadow_mmap_threshold;
+  if (SANITIZER_LINUX &&
+      UNLIKELY(page_end >= page_start + threshold && tag == 0)) {
+    internal_memset((void *)shadow_start, tag, page_start - shadow_start);
+    internal_memset((void *)page_end, tag,
+                    shadow_start + shadow_size - page_end);
+    // For an anonymous private mapping MADV_DONTNEED will return a zero page on
+    // Linux.
+    ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end);
+  } else {
+    internal_memset((void *)shadow_start, tag, shadow_size);
+  }
+  return AddTagToPointer(p, tag);
 }
 
+} // namespace __hwasan
+
 #endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
diff --git a/compiler-rt/lib/hwasan/hwasan_mapping.h b/compiler-rt/lib/hwasan/hwasan_mapping.h
index c149687bdfa6..79a143632f6a 100644
--- a/compiler-rt/lib/hwasan/hwasan_mapping.h
+++ b/compiler-rt/lib/hwasan/hwasan_mapping.h
@@ -48,12 +48,14 @@ extern uptr kHighShadowEnd;
 extern uptr kHighMemStart;
 extern uptr kHighMemEnd;
 
+inline uptr GetShadowOffset() {
+  return SANITIZER_FUCHSIA ? 0 : __hwasan_shadow_memory_dynamic_address;
+}
 inline uptr MemToShadow(uptr untagged_addr) {
-  return (untagged_addr >> kShadowScale) +
-         __hwasan_shadow_memory_dynamic_address;
+  return (untagged_addr >> kShadowScale) + GetShadowOffset();
 }
 inline uptr ShadowToMem(uptr shadow_addr) {
-  return (shadow_addr - __hwasan_shadow_memory_dynamic_address) << kShadowScale;
+  return (shadow_addr - GetShadowOffset()) << kShadowScale;
 }
 inline uptr MemToShadowSize(uptr size) {
   return size >> kShadowScale;
@@ -61,6 +63,13 @@ inline uptr MemToShadowSize(uptr size) {
 
 bool MemIsApp(uptr p);
 
+inline bool MemIsShadow(uptr p) {
+  return (kLowShadowStart <= p && p <= kLowShadowEnd) ||
+         (kHighShadowStart <= p && p <= kHighShadowEnd);
+}
+
+uptr GetAliasRegionStart();
+
 }  // namespace __hwasan
 
 #endif  // HWASAN_MAPPING_H
diff --git a/compiler-rt/lib/hwasan/hwasan_memintrinsics.cpp b/compiler-rt/lib/hwasan/hwasan_memintrinsics.cpp
index e82d77a1bc16..fab017aae60b 100644
--- a/compiler-rt/lib/hwasan/hwasan_memintrinsics.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_memintrinsics.cpp
@@ -24,7 +24,7 @@ using namespace __hwasan;
 void *__hwasan_memset(void *block, int c, uptr size) {
   CheckAddressSized<ErrorAction::Recover, AccessType::Store>(
       reinterpret_cast<uptr>(block), size);
-  return memset(UntagPtr(block), c, size);
+  return memset(block, c, size);
 }
 
 void *__hwasan_memcpy(void *to, const void *from, uptr size) {
@@ -32,7 +32,7 @@ void *__hwasan_memcpy(void *to, const void *from, uptr size) {
       reinterpret_cast<uptr>(to), size);
   CheckAddressSized<ErrorAction::Recover, AccessType::Load>(
       reinterpret_cast<uptr>(from), size);
-  return memcpy(UntagPtr(to), UntagPtr(from), size);
+  return memcpy(to, from, size);
 }
 
 void *__hwasan_memmove(void *to, const void *from, uptr size) {
diff --git a/compiler-rt/lib/hwasan/hwasan_new_delete.cpp b/compiler-rt/lib/hwasan/hwasan_new_delete.cpp
index 8d01d3944f2b..4e057a651e1d 100644
--- a/compiler-rt/lib/hwasan/hwasan_new_delete.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_new_delete.cpp
@@ -27,6 +27,12 @@
   void *res = hwasan_malloc(size, &stack);\
   if (!nothrow && UNLIKELY(!res)) ReportOutOfMemory(size, &stack);\
   return res
+#define OPERATOR_NEW_ALIGN_BODY(nothrow)                                    \
+  GET_MALLOC_STACK_TRACE;                                                   \
+  void *res = hwasan_aligned_alloc(static_cast<uptr>(align), size, &stack); \
+  if (!nothrow && UNLIKELY(!res))                                           \
+    ReportOutOfMemory(size, &stack);                                        \
+  return res
 
 #define OPERATOR_DELETE_BODY \
   GET_MALLOC_STACK_TRACE; \
@@ -67,15 +73,63 @@ void *operator new[](size_t size, std::nothrow_t const&) {
   OPERATOR_NEW_BODY(true /*nothrow*/);
 }
 
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; }
-INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-void operator delete[](void *ptr, std::nothrow_t const&) {
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(void *ptr)
+    NOEXCEPT {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
+    void *ptr) NOEXCEPT {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
+    void *ptr, std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
+    void *ptr, std::nothrow_t const &) {
+  OPERATOR_DELETE_BODY;
+}
+
+#endif  // OPERATOR_NEW_BODY
+
+#ifdef OPERATOR_NEW_ALIGN_BODY
+
+namespace std {
+enum class align_val_t : size_t {};
+}  // namespace std
+
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(
+    size_t size, std::align_val_t align) {
+  OPERATOR_NEW_ALIGN_BODY(false /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](
+    size_t size, std::align_val_t align) {
+  OPERATOR_NEW_ALIGN_BODY(false /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new(
+    size_t size, std::align_val_t align, std::nothrow_t const &) {
+  OPERATOR_NEW_ALIGN_BODY(true /*nothrow*/);
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](
+    size_t size, std::align_val_t align, std::nothrow_t const &) {
+  OPERATOR_NEW_ALIGN_BODY(true /*nothrow*/);
+}
+
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
+    void *ptr, std::align_val_t align) NOEXCEPT {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
+    void *ptr, std::align_val_t) NOEXCEPT {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(
+    void *ptr, std::align_val_t, std::nothrow_t const &) NOEXCEPT {
+  OPERATOR_DELETE_BODY;
+}
+INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[](
+    void *ptr, std::align_val_t, std::nothrow_t const &) NOEXCEPT {
   OPERATOR_DELETE_BODY;
 }
 
-#endif // OPERATOR_NEW_BODY
+#endif  // OPERATOR_NEW_ALIGN_BODY
diff --git a/compiler-rt/lib/hwasan/hwasan_poisoning.cpp b/compiler-rt/lib/hwasan/hwasan_poisoning.cpp
index 2a0816428e75..5aafdb1884b5 100644
--- a/compiler-rt/lib/hwasan/hwasan_poisoning.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_poisoning.cpp
@@ -19,30 +19,6 @@
 
 namespace __hwasan {
 
-uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
-  CHECK(IsAligned(p, kShadowAlignment));
-  CHECK(IsAligned(size, kShadowAlignment));
-  uptr shadow_start = MemToShadow(p);
-  uptr shadow_size = MemToShadowSize(size);
-
-  uptr page_size = GetPageSizeCached();
-  uptr page_start = RoundUpTo(shadow_start, page_size);
-  uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size);
-  uptr threshold = common_flags()->clear_shadow_mmap_threshold;
-  if (SANITIZER_LINUX &&
-      UNLIKELY(page_end >= page_start + threshold && tag == 0)) {
-    internal_memset((void *)shadow_start, tag, page_start - shadow_start);
-    internal_memset((void *)page_end, tag,
-                    shadow_start + shadow_size - page_end);
-    // For an anonymous private mapping MADV_DONTNEED will return a zero page on
-    // Linux.
-    ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end);
-  } else {
-    internal_memset((void *)shadow_start, tag, shadow_size);
-  }
-  return AddTagToPointer(p, tag);
-}
-
 uptr TagMemory(uptr p, uptr size, tag_t tag) {
   uptr start = RoundDownTo(p, kShadowAlignment);
   uptr end = RoundUpTo(p + size, kShadowAlignment);
diff --git a/compiler-rt/lib/hwasan/hwasan_report.cpp b/compiler-rt/lib/hwasan/hwasan_report.cpp
index 4448d9243767..44047c9fdaf8 100644
--- a/compiler-rt/lib/hwasan/hwasan_report.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_report.cpp
@@ -224,7 +224,7 @@ static void PrintStackAllocations(StackAllocationsRingBuffer *sa,
 
   // We didn't find any locals. Most likely we don't have symbols, so dump
   // the information that we have for offline analysis.
-  InternalScopedString frame_desc(GetPageSizeCached() * 2);
+  InternalScopedString frame_desc;
   Printf("Previously allocated frames:\n");
   for (uptr i = 0; i < frames; i++) {
     const uptr *record_addr = &(*sa)[i];
@@ -236,12 +236,12 @@ static void PrintStackAllocations(StackAllocationsRingBuffer *sa,
     frame_desc.append("  record_addr:0x%zx record:0x%zx",
                       reinterpret_cast<uptr>(record_addr), record);
     if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) {
-      RenderFrame(&frame_desc, " %F %L\n", 0, frame->info.address, &frame->info,
+      RenderFrame(&frame_desc, " %F %L", 0, frame->info.address, &frame->info,
                   common_flags()->symbolize_vs_style,
                   common_flags()->strip_path_prefix);
       frame->ClearAll();
     }
-    Printf("%s", frame_desc.data());
+    Printf("%s\n", frame_desc.data());
     frame_desc.clear();
   }
 }
@@ -296,6 +296,75 @@ static uptr GetGlobalSizeFromDescriptor(uptr ptr) {
   return 0;
 }
 
+static void ShowHeapOrGlobalCandidate(uptr untagged_addr, tag_t *candidate,
+                                      tag_t *left, tag_t *right) {
+  Decorator d;
+  uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate));
+  HwasanChunkView chunk = FindHeapChunkByAddress(mem);
+  if (chunk.IsAllocated()) {
+    uptr offset;
+    const char *whence;
+    if (untagged_addr < chunk.End() && untagged_addr >= chunk.Beg()) {
+      offset = untagged_addr - chunk.Beg();
+      whence = "inside";
+    } else if (candidate == left) {
+      offset = untagged_addr - chunk.End();
+      whence = "to the right of";
+    } else {
+      offset = chunk.Beg() - untagged_addr;
+      whence = "to the left of";
+    }
+    Printf("%s", d.Error());
+    Printf("\nCause: heap-buffer-overflow\n");
+    Printf("%s", d.Default());
+    Printf("%s", d.Location());
+    Printf("%p is located %zd bytes %s %zd-byte region [%p,%p)\n",
+           untagged_addr, offset, whence, chunk.UsedSize(), chunk.Beg(),
+           chunk.End());
+    Printf("%s", d.Allocation());
+    Printf("allocated here:\n");
+    Printf("%s", d.Default());
+    GetStackTraceFromId(chunk.GetAllocStackId()).Print();
+    return;
+  }
+  // Check whether the address points into a loaded library. If so, this is
+  // most likely a global variable.
+  const char *module_name;
+  uptr module_address;
+  Symbolizer *sym = Symbolizer::GetOrInit();
+  if (sym->GetModuleNameAndOffsetForPC(mem, &module_name, &module_address)) {
+    Printf("%s", d.Error());
+    Printf("\nCause: global-overflow\n");
+    Printf("%s", d.Default());
+    DataInfo info;
+    Printf("%s", d.Location());
+    if (sym->SymbolizeData(mem, &info) && info.start) {
+      Printf(
+          "%p is located %zd bytes to the %s of %zd-byte global variable "
+          "%s [%p,%p) in %s\n",
+          untagged_addr,
+          candidate == left ? untagged_addr - (info.start + info.size)
+                            : info.start - untagged_addr,
+          candidate == left ? "right" : "left", info.size, info.name,
+          info.start, info.start + info.size, module_name);
+    } else {
+      uptr size = GetGlobalSizeFromDescriptor(mem);
+      if (size == 0)
+        // We couldn't find the size of the global from the descriptors.
+        Printf("%p is located to the %s of a global variable in (%s+0x%x)\n",
+               untagged_addr, candidate == left ? "right" : "left", module_name,
+               module_address);
+      else
+        Printf(
+            "%p is located to the %s of a %zd-byte global variable in "
+            "(%s+0x%x)\n",
+            untagged_addr, candidate == left ? "right" : "left", size,
+            module_name, module_address);
+    }
+    Printf("%s", d.Default());
+  }
+}
+
 void PrintAddressDescription(
     uptr tagged_addr, uptr access_size,
     StackAllocationsRingBuffer *current_stack_allocations) {
@@ -317,78 +386,59 @@ void PrintAddressDescription(
            d.Default());
   }
 
+  tag_t addr_tag = GetTagFromPointer(tagged_addr);
+
+  bool on_stack = false;
+  // Check stack first. If the address is on the stack of a live thread, we
+  // know it cannot be a heap / global overflow.
+  hwasanThreadList().VisitAllLiveThreads([&](Thread *t) {
+    if (t->AddrIsInStack(untagged_addr)) {
+      on_stack = true;
+      // TODO(fmayer): figure out how to distinguish use-after-return and
+      // stack-buffer-overflow.
+      Printf("%s", d.Error());
+      Printf("\nCause: stack tag-mismatch\n");
+      Printf("%s", d.Location());
+      Printf("Address %p is located in stack of thread T%zd\n", untagged_addr,
+             t->unique_id());
+      Printf("%s", d.Default());
+      t->Announce();
+
+      auto *sa = (t == GetCurrentThread() && current_stack_allocations)
+                     ? current_stack_allocations
+                     : t->stack_allocations();
+      PrintStackAllocations(sa, addr_tag, untagged_addr);
+      num_descriptions_printed++;
+    }
+  });
+
   // Check if this looks like a heap buffer overflow by scanning
   // the shadow left and right and looking for the first adjacent
   // object with a different memory tag. If that tag matches addr_tag,
   // check the allocator if it has a live chunk there.
-  tag_t addr_tag = GetTagFromPointer(tagged_addr);
   tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr));
   tag_t *candidate = nullptr, *left = tag_ptr, *right = tag_ptr;
-  for (int i = 0; i < 1000; i++) {
-    if (TagsEqual(addr_tag, left)) {
+  uptr candidate_distance = 0;
+  for (; candidate_distance < 1000; candidate_distance++) {
+    if (MemIsShadow(reinterpret_cast<uptr>(left)) &&
+        TagsEqual(addr_tag, left)) {
       candidate = left;
       break;
     }
     --left;
-    if (TagsEqual(addr_tag, right)) {
+    if (MemIsShadow(reinterpret_cast<uptr>(right)) &&
+        TagsEqual(addr_tag, right)) {
       candidate = right;
       break;
     }
     ++right;
   }
 
-  if (candidate) {
-    uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate));
-    HwasanChunkView chunk = FindHeapChunkByAddress(mem);
-    if (chunk.IsAllocated()) {
-      Printf("%s", d.Location());
-      Printf("%p is located %zd bytes to the %s of %zd-byte region [%p,%p)\n",
-             untagged_addr,
-             candidate == left ? untagged_addr - chunk.End()
-                               : chunk.Beg() - untagged_addr,
-             candidate == left ? "right" : "left", chunk.UsedSize(),
-             chunk.Beg(), chunk.End());
-      Printf("%s", d.Allocation());
-      Printf("allocated here:\n");
-      Printf("%s", d.Default());
-      GetStackTraceFromId(chunk.GetAllocStackId()).Print();
-      num_descriptions_printed++;
-    } else {
-      // Check whether the address points into a loaded library. If so, this is
-      // most likely a global variable.
-      const char *module_name;
-      uptr module_address;
-      Symbolizer *sym = Symbolizer::GetOrInit();
-      if (sym->GetModuleNameAndOffsetForPC(mem, &module_name,
-                                           &module_address)) {
-        DataInfo info;
-        if (sym->SymbolizeData(mem, &info) && info.start) {
-          Printf(
-              "%p is located %zd bytes to the %s of %zd-byte global variable "
-              "%s [%p,%p) in %s\n",
-              untagged_addr,
-              candidate == left ? untagged_addr - (info.start + info.size)
-                                : info.start - untagged_addr,
-              candidate == left ? "right" : "left", info.size, info.name,
-              info.start, info.start + info.size, module_name);
-        } else {
-          uptr size = GetGlobalSizeFromDescriptor(mem);
-          if (size == 0)
-            // We couldn't find the size of the global from the descriptors.
-            Printf(
-                "%p is located to the %s of a global variable in (%s+0x%x)\n",
-                untagged_addr, candidate == left ? "right" : "left",
-                module_name, module_address);
-          else
-            Printf(
-                "%p is located to the %s of a %zd-byte global variable in "
-                "(%s+0x%x)\n",
-                untagged_addr, candidate == left ? "right" : "left", size,
-                module_name, module_address);
-        }
-        num_descriptions_printed++;
-      }
-    }
+  constexpr auto kCloseCandidateDistance = 1;
+
+  if (!on_stack && candidate && candidate_distance <= kCloseCandidateDistance) {
+    ShowHeapOrGlobalCandidate(untagged_addr, candidate, left, right);
+    num_descriptions_printed++;
   }
 
   hwasanThreadList().VisitAllLiveThreads([&](Thread *t) {
@@ -398,6 +448,8 @@ void PrintAddressDescription(
     if (FindHeapAllocation(t->heap_allocations(), tagged_addr, &har,
                            &ring_index, &num_matching_addrs,
                            &num_matching_addrs_4b)) {
+      Printf("%s", d.Error());
+      Printf("\nCause: use-after-free\n");
       Printf("%s", d.Location());
       Printf("%p is located %zd bytes inside of %zd-byte region [%p,%p)\n",
              untagged_addr, untagged_addr - UntagAddr(har.tagged_addr),
@@ -424,29 +476,25 @@ void PrintAddressDescription(
       t->Announce();
       num_descriptions_printed++;
     }
-
-    // Very basic check for stack memory.
-    if (t->AddrIsInStack(untagged_addr)) {
-      Printf("%s", d.Location());
-      Printf("Address %p is located in stack of thread T%zd\n", untagged_addr,
-             t->unique_id());
-      Printf("%s", d.Default());
-      t->Announce();
-
-      auto *sa = (t == GetCurrentThread() && current_stack_allocations)
-                     ? current_stack_allocations
-                     : t->stack_allocations();
-      PrintStackAllocations(sa, addr_tag, untagged_addr);
-      num_descriptions_printed++;
-    }
   });
 
+  if (candidate && num_descriptions_printed == 0) {
+    ShowHeapOrGlobalCandidate(untagged_addr, candidate, left, right);
+    num_descriptions_printed++;
+  }
+
   // Print the remaining threads, as an extra information, 1 line per thread.
   hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { t->Announce(); });
 
   if (!num_descriptions_printed)
     // We exhausted our possibilities. Bail out.
     Printf("HWAddressSanitizer can not describe address in more detail.\n");
+  if (num_descriptions_printed > 1) {
+    Printf(
+        "There are %d potential causes, printed above in order "
+        "of likeliness.\n",
+        num_descriptions_printed);
+  }
 }
 
 void ReportStats() {}
@@ -459,7 +507,7 @@ static void PrintTagInfoAroundAddr(tag_t *tag_ptr, uptr num_rows,
       RoundDownTo(reinterpret_cast<uptr>(tag_ptr), row_len));
   tag_t *beg_row = center_row_beg - row_len * (num_rows / 2);
   tag_t *end_row = center_row_beg + row_len * ((num_rows + 1) / 2);
-  InternalScopedString s(GetPageSizeCached() * 8);
+  InternalScopedString s;
   for (tag_t *row = beg_row; row < end_row; row += row_len) {
     s.append("%s", row == center_row_beg ? "=>" : "  ");
     s.append("%p:", row);
@@ -538,6 +586,12 @@ void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size,
   Report("ERROR: %s: %s; heap object [%p,%p) of size %zd\n", SanitizerToolName,
          bug_type, untagged_addr, untagged_addr + orig_size, orig_size);
   Printf("\n%s", d.Default());
+  Printf(
+      "Stack of invalid access unknown. Issue detected at deallocation "
+      "time.\n");
+  Printf("%s", d.Allocation());
+  Printf("deallocated here:\n");
+  Printf("%s", d.Default());
   stack->Print();
   HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr);
   if (chunk.Beg()) {
@@ -547,7 +601,7 @@ void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size,
     GetStackTraceFromId(chunk.GetAllocStackId()).Print();
   }
 
-  InternalScopedString s(GetPageSizeCached() * 8);
+  InternalScopedString s;
   CHECK_GT(tail_size, 0U);
   CHECK_LT(tail_size, kShadowAlignment);
   u8 *tail = reinterpret_cast<u8*>(untagged_addr + orig_size);
@@ -657,8 +711,10 @@ void ReportRegisters(uptr *frame, uptr pc) {
        frame[20], frame[21], frame[22], frame[23]);
   Printf("    x24 %016llx  x25 %016llx  x26 %016llx  x27 %016llx\n",
        frame[24], frame[25], frame[26], frame[27]);
-  Printf("    x28 %016llx  x29 %016llx  x30 %016llx\n",
-       frame[28], frame[29], frame[30]);
+  // hwasan_check* reduces the stack pointer by 256, then __hwasan_tag_mismatch
+  // passes it to this function.
+  Printf("    x28 %016llx  x29 %016llx  x30 %016llx   sp %016llx\n", frame[28],
+         frame[29], frame[30], reinterpret_cast<u8 *>(frame) + 256);
 }
 
 }  // namespace __hwasan
diff --git a/compiler-rt/lib/hwasan/hwasan_setjmp.S b/compiler-rt/lib/hwasan/hwasan_setjmp.S
index 0c1354331940..381af63363cc 100644
--- a/compiler-rt/lib/hwasan/hwasan_setjmp.S
+++ b/compiler-rt/lib/hwasan/hwasan_setjmp.S
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "sanitizer_common/sanitizer_asm.h"
+#include "builtins/assembly.h"
 
 #if HWASAN_WITH_INTERCEPTORS && defined(__aarch64__)
 #include "sanitizer_common/sanitizer_platform.h"
@@ -34,6 +35,7 @@
 ASM_TYPE_FUNCTION(__interceptor_setjmp)
 __interceptor_setjmp:
   CFI_STARTPROC
+  BTI_C
   mov	x1, #0
   b	__interceptor_sigsetjmp
   CFI_ENDPROC
@@ -46,6 +48,7 @@ ASM_SIZE(__interceptor_setjmp)
 ASM_TYPE_FUNCTION(__interceptor_setjmp_bionic)
 __interceptor_setjmp_bionic:
   CFI_STARTPROC
+  BTI_C
   mov	x1, #1
   b	__interceptor_sigsetjmp
   CFI_ENDPROC
@@ -56,6 +59,7 @@ ASM_SIZE(__interceptor_setjmp_bionic)
 ASM_TYPE_FUNCTION(__interceptor_sigsetjmp)
 __interceptor_sigsetjmp:
   CFI_STARTPROC
+  BTI_C
   stp	x19, x20, [x0, #0<<3]
   stp	x21, x22, [x0, #2<<3]
   stp	x23, x24, [x0, #4<<3]
@@ -98,3 +102,5 @@ ALIAS __interceptor_setjmp, _setjmp
 
 // We do not need executable stack.
 NO_EXEC_STACK_DIRECTIVE
+
+GNU_PROPERTY_BTI_PAC
diff --git a/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S
index 08df12736bb4..bcb0df420190 100644
--- a/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S
+++ b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S
@@ -1,4 +1,5 @@
 #include "sanitizer_common/sanitizer_asm.h"
+#include "builtins/assembly.h"
 
 // The content of this file is AArch64-only:
 #if defined(__aarch64__)
@@ -74,6 +75,8 @@
 .global __hwasan_tag_mismatch
 .type __hwasan_tag_mismatch, %function
 __hwasan_tag_mismatch:
+  BTI_J
+
   // Compute the granule position one past the end of the access.
   mov x16, #1
   and x17, x1, #0xf
@@ -106,6 +109,7 @@ __hwasan_tag_mismatch:
 .type __hwasan_tag_mismatch_v2, %function
 __hwasan_tag_mismatch_v2:
   CFI_STARTPROC
+  BTI_J
 
   // Set the CFA to be the return address for caller of __hwasan_check_*. Note
   // that we do not emit CFI predicates to describe the contents of this stack
@@ -150,3 +154,5 @@ __hwasan_tag_mismatch_v2:
 
 // We do not need executable stack.
 NO_EXEC_STACK_DIRECTIVE
+
+GNU_PROPERTY_BTI_PAC
diff --git a/compiler-rt/lib/hwasan/hwasan_thread.cpp b/compiler-rt/lib/hwasan/hwasan_thread.cpp
index b81a6350c05c..ee747a3beea5 100644
--- a/compiler-rt/lib/hwasan/hwasan_thread.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_thread.cpp
@@ -34,12 +34,28 @@ void Thread::InitRandomState() {
     stack_allocations_->push(0);
 }
 
-void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) {
+void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size,
+                  const InitState *state) {
+  CHECK_EQ(0, unique_id_);  // try to catch bad stack reuse
+  CHECK_EQ(0, stack_top_);
+  CHECK_EQ(0, stack_bottom_);
+
   static u64 unique_id;
   unique_id_ = unique_id++;
   if (auto sz = flags()->heap_history_size)
     heap_allocations_ = HeapAllocationsRingBuffer::New(sz);
 
+  InitStackAndTls(state);
+#if !SANITIZER_FUCHSIA
+  // Do not initialize the stack ring buffer just yet on Fuchsia. Threads will
+  // be initialized before we enter the thread itself, so we will instead call
+  // this later.
+  InitStackRingBuffer(stack_buffer_start, stack_buffer_size);
+#endif
+}
+
+void Thread::InitStackRingBuffer(uptr stack_buffer_start,
+                                 uptr stack_buffer_size) {
   HwasanTSDThreadInit();  // Only needed with interceptors.
   uptr *ThreadLong = GetCurrentThreadLongPtr();
   // The following implicitly sets (this) as the current thread.
@@ -51,13 +67,6 @@ void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) {
   // ScopedTaggingDisable needs GetCurrentThread to be set up.
   ScopedTaggingDisabler disabler;
 
-  uptr tls_size;
-  uptr stack_size;
-  GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_,
-                       &tls_size);
-  stack_top_ = stack_bottom_ + stack_size;
-  tls_end_ = tls_begin_ + tls_size;
-
   if (stack_bottom_) {
     int local;
     CHECK(AddrIsInStack((uptr)&local));
@@ -113,18 +122,21 @@ static u32 xorshift(u32 state) {
 }
 
 // Generate a (pseudo-)random non-zero tag.
-tag_t Thread::GenerateRandomTag() {
+tag_t Thread::GenerateRandomTag(uptr num_bits) {
+  DCHECK_GT(num_bits, 0);
   if (tagging_disabled_) return 0;
   tag_t tag;
+  const uptr tag_mask = (1ULL << num_bits) - 1;
   do {
     if (flags()->random_tags) {
       if (!random_buffer_)
         random_buffer_ = random_state_ = xorshift(random_state_);
       CHECK(random_buffer_);
-      tag = random_buffer_ & 0xFF;
-      random_buffer_ >>= 8;
+      tag = random_buffer_ & tag_mask;
+      random_buffer_ >>= num_bits;
     } else {
-      tag = random_state_ = (random_state_ + 1) & 0xFF;
+      random_state_ += 1;
+      tag = random_state_ & tag_mask;
     }
   } while (!tag);
   return tag;
diff --git a/compiler-rt/lib/hwasan/hwasan_thread.h b/compiler-rt/lib/hwasan/hwasan_thread.h
index 88958daf767c..9f20afe1dc76 100644
--- a/compiler-rt/lib/hwasan/hwasan_thread.h
+++ b/compiler-rt/lib/hwasan/hwasan_thread.h
@@ -23,8 +23,17 @@ typedef __sanitizer::CompactRingBuffer<uptr> StackAllocationsRingBuffer;
 
 class Thread {
  public:
-  void Init(uptr stack_buffer_start, uptr stack_buffer_size);  // Must be called from the thread itself.
+  // These are optional parameters that can be passed to Init.
+  struct InitState;
+
+  void Init(uptr stack_buffer_start, uptr stack_buffer_size,
+            const InitState *state = nullptr);
   void InitRandomState();
+  void InitStackAndTls(const InitState *state = nullptr);
+
+  // Must be called from the thread itself.
+  void InitStackRingBuffer(uptr stack_buffer_start, uptr stack_buffer_size);
+
   void Destroy();
 
   uptr stack_top() { return stack_top_; }
@@ -42,7 +51,7 @@ class Thread {
   HeapAllocationsRingBuffer *heap_allocations() { return heap_allocations_; }
   StackAllocationsRingBuffer *stack_allocations() { return stack_allocations_; }
 
-  tag_t GenerateRandomTag();
+  tag_t GenerateRandomTag(uptr num_bits = kTagBits);
 
   void DisableTagging() { tagging_disabled_++; }
   void EnableTagging() { tagging_disabled_--; }
diff --git a/compiler-rt/lib/hwasan/hwasan_thread_list.cpp b/compiler-rt/lib/hwasan/hwasan_thread_list.cpp
index a31eee84ed93..fa46e658b69d 100644
--- a/compiler-rt/lib/hwasan/hwasan_thread_list.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_thread_list.cpp
@@ -12,4 +12,4 @@ void InitThreadList(uptr storage, uptr size) {
       new (thread_list_placeholder) HwasanThreadList(storage, size);
 }
 
-} // namespace
+} // namespace __hwasan
diff --git a/compiler-rt/lib/hwasan/hwasan_thread_list.h b/compiler-rt/lib/hwasan/hwasan_thread_list.h
index e596bde36662..15916a802d6e 100644
--- a/compiler-rt/lib/hwasan/hwasan_thread_list.h
+++ b/compiler-rt/lib/hwasan/hwasan_thread_list.h
@@ -85,21 +85,26 @@ class HwasanThreadList {
         RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2);
   }
 
-  Thread *CreateCurrentThread() {
-    Thread *t;
+  Thread *CreateCurrentThread(const Thread::InitState *state = nullptr) {
+    Thread *t = nullptr;
     {
-      SpinMutexLock l(&list_mutex_);
+      SpinMutexLock l(&free_list_mutex_);
       if (!free_list_.empty()) {
         t = free_list_.back();
         free_list_.pop_back();
-        uptr start = (uptr)t - ring_buffer_size_;
-        internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread));
-      } else {
-        t = AllocThread();
       }
+    }
+    if (t) {
+      uptr start = (uptr)t - ring_buffer_size_;
+      internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread));
+    } else {
+      t = AllocThread();
+    }
+    {
+      SpinMutexLock l(&live_list_mutex_);
       live_list_.push_back(t);
     }
-    t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_);
+    t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_, state);
     AddThreadStats(t);
     return t;
   }
@@ -110,6 +115,7 @@ class HwasanThreadList {
   }
 
   void RemoveThreadFromLiveList(Thread *t) {
+    SpinMutexLock l(&live_list_mutex_);
     for (Thread *&t2 : live_list_)
       if (t2 == t) {
         // To remove t2, copy the last element of the list in t2's position, and
@@ -124,10 +130,10 @@ class HwasanThreadList {
   void ReleaseThread(Thread *t) {
     RemoveThreadStats(t);
     t->Destroy();
-    SpinMutexLock l(&list_mutex_);
+    DontNeedThread(t);
     RemoveThreadFromLiveList(t);
+    SpinMutexLock l(&free_list_mutex_);
     free_list_.push_back(t);
-    DontNeedThread(t);
   }
 
   Thread *GetThreadByBufferAddress(uptr p) {
@@ -144,7 +150,7 @@ class HwasanThreadList {
 
   template <class CB>
   void VisitAllLiveThreads(CB cb) {
-    SpinMutexLock l(&list_mutex_);
+    SpinMutexLock l(&live_list_mutex_);
     for (Thread *t : live_list_) cb(t);
   }
 
@@ -165,8 +171,11 @@ class HwasanThreadList {
     return stats_;
   }
 
+  uptr GetRingBufferSize() const { return ring_buffer_size_; }
+
  private:
   Thread *AllocThread() {
+    SpinMutexLock l(&free_space_mutex_);
     uptr align = ring_buffer_size_ * 2;
     CHECK(IsAligned(free_space_, align));
     Thread *t = (Thread *)(free_space_ + ring_buffer_size_);
@@ -175,14 +184,16 @@ class HwasanThreadList {
     return t;
   }
 
+  SpinMutex free_space_mutex_;
   uptr free_space_;
   uptr free_space_end_;
   uptr ring_buffer_size_;
   uptr thread_alloc_size_;
 
+  SpinMutex free_list_mutex_;
   InternalMmapVector<Thread *> free_list_;
+  SpinMutex live_list_mutex_;
   InternalMmapVector<Thread *> live_list_;
-  SpinMutex list_mutex_;
 
   ThreadStats stats_;
   SpinMutex stats_mutex_;
@@ -191,4 +202,4 @@ class HwasanThreadList {
 void InitThreadList(uptr storage, uptr size);
 HwasanThreadList &hwasanThreadList();
 
-} // namespace
+} // namespace __hwasan