Vendor import of compiler-rt trunk r307894:vendor/compiler-rt/compiler-rt-trunk-r307894

https://llvm.org/svn/llvm-project/compiler-rt/trunk@307894

91 files changed, 1453 insertions, 1007 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3152233587c6..3195de1e5d1e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -80,7 +80,7 @@ pythonize_bool(COMPILER_RT_DEBUG)
 
 include(config-ix)
 
-if(APPLE AND SANITIZER_MIN_OSX_VERSION VERSION_LESS "10.9")
+if(APPLE AND SANITIZER_MIN_OSX_VERSION AND SANITIZER_MIN_OSX_VERSION VERSION_LESS "10.9")
   # Mac OS X prior to 10.9 had problems with exporting symbols from
   # libc++/libc++abi.
   set(use_cxxabi_default OFF)
diff --git a/cmake/Modules/CompilerRTDarwinUtils.cmake b/cmake/Modules/CompilerRTDarwinUtils.cmake
index baea4a067fa8..f646975475bb 100644
--- a/cmake/Modules/CompilerRTDarwinUtils.cmake
+++ b/cmake/Modules/CompilerRTDarwinUtils.cmake
@@ -4,14 +4,23 @@ include(CMakeParseArguments)
 # set the default Xcode to use. This function finds the SDKs that are present in
 # the current Xcode.
 function(find_darwin_sdk_dir var sdk_name)
-  # Let's first try the internal SDK, otherwise use the public SDK.
-  execute_process(
-    COMMAND xcodebuild -version -sdk ${sdk_name}.internal Path
-    RESULT_VARIABLE result_process
-    OUTPUT_VARIABLE var_internal
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-    ERROR_FILE /dev/null
-  )
+  set(DARWIN_${sdk_name}_CACHED_SYSROOT "" CACHE STRING "Darwin SDK path for SDK ${sdk_name}.")
+  set(DARWIN_PREFER_PUBLIC_SDK OFF CACHE BOOL "Prefer Darwin public SDK, even when an internal SDK is present.")
+
+  if(DARWIN_${sdk_name}_CACHED_SYSROOT)
+    set(${var} ${DARWIN_${sdk_name}_CACHED_SYSROOT} PARENT_SCOPE)
+    return()
+  endif()
+  if(NOT DARWIN_PREFER_PUBLIC_SDK)
+    # Let's first try the internal SDK, otherwise use the public SDK.
+    execute_process(
+      COMMAND xcodebuild -version -sdk ${sdk_name}.internal Path
+      RESULT_VARIABLE result_process
+      OUTPUT_VARIABLE var_internal
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      ERROR_FILE /dev/null
+    )
+  endif()
   if((NOT result_process EQUAL 0) OR "" STREQUAL "${var_internal}")
     execute_process(
       COMMAND xcodebuild -version -sdk ${sdk_name} Path
@@ -26,6 +35,7 @@ function(find_darwin_sdk_dir var sdk_name)
   if(result_process EQUAL 0)
     set(${var} ${var_internal} PARENT_SCOPE)
   endif()
+  set(DARWIN_${sdk_name}_CACHED_SYSROOT ${var_internal} CACHE STRING "Darwin SDK path for SDK ${sdk_name}." FORCE)
 endfunction()
 
 # There isn't a clear mapping of what architectures are supported with a given
diff --git a/cmake/config-ix.cmake b/cmake/config-ix.cmake
index 0da98b9a489d..c329c6a979a3 100644
--- a/cmake/config-ix.cmake
+++ b/cmake/config-ix.cmake
@@ -303,9 +303,7 @@ if(APPLE)
         if(DARWIN_${platform}sim_ARCHS)
           list(APPEND SANITIZER_COMMON_SUPPORTED_OS ${platform}sim)
           list(APPEND PROFILE_SUPPORTED_OS ${platform}sim)
-          if(DARWIN_${platform}_SYSROOT_INTERNAL)
-            list(APPEND TSAN_SUPPORTED_OS ${platform}sim)
-          endif()
+          list(APPEND TSAN_SUPPORTED_OS ${platform}sim)
         endif()
         foreach(arch ${DARWIN_${platform}sim_ARCHS})
           list(APPEND COMPILER_RT_SUPPORTED_ARCH ${arch})
@@ -330,6 +328,7 @@ if(APPLE)
         if(DARWIN_${platform}_ARCHS)
           list(APPEND SANITIZER_COMMON_SUPPORTED_OS ${platform})
           list(APPEND PROFILE_SUPPORTED_OS ${platform})
+          list(APPEND TSAN_SUPPORTED_OS ${platform})
         endif()
         foreach(arch ${DARWIN_${platform}_ARCHS})
           list(APPEND COMPILER_RT_SUPPORTED_ARCH ${arch})
diff --git a/lib/asan/asan_errors.cc b/lib/asan/asan_errors.cc
index 57490ad180b5..b7a38eb7cece 100644
--- a/lib/asan/asan_errors.cc
+++ b/lib/asan/asan_errors.cc
@@ -61,10 +61,9 @@ static void MaybeDumpRegisters(void *context) {
 static void MaybeReportNonExecRegion(uptr pc) {
 #if SANITIZER_FREEBSD || SANITIZER_LINUX
   MemoryMappingLayout proc_maps(/*cache_enabled*/ true);
-  uptr start, end, protection;
-  while (proc_maps.Next(&start, &end, nullptr, nullptr, 0, &protection)) {
-    if (pc >= start && pc < end &&
-        !(protection & MemoryMappingLayout::kProtectionExecute))
+  MemoryMappedSegment segment;
+  while (proc_maps.Next(&segment)) {
+    if (pc >= segment.start && pc < segment.end && !segment.IsExecutable())
       Report("Hint: PC is at a non-executable region. Maybe a wild jump?\n");
   }
 #endif
diff --git a/lib/asan/asan_internal.h b/lib/asan/asan_internal.h
index 3b70695249e4..f09bbd83af25 100644
--- a/lib/asan/asan_internal.h
+++ b/lib/asan/asan_internal.h
@@ -75,6 +75,7 @@ void NORETURN ShowStatsAndAbort();
 void ReplaceSystemMalloc();
 
 // asan_linux.cc / asan_mac.cc / asan_win.cc
+uptr FindDynamicShadowStart();
 void *AsanDoesNotSupportStaticLinkage();
 void AsanCheckDynamicRTPrereqs();
 void AsanCheckIncompatibleRT();
diff --git a/lib/asan/asan_linux.cc b/lib/asan/asan_linux.cc
index 50ef84c39a66..6d47ba432a61 100644
--- a/lib/asan/asan_linux.cc
+++ b/lib/asan/asan_linux.cc
@@ -77,6 +77,11 @@ void *AsanDoesNotSupportStaticLinkage() {
   return &_DYNAMIC;  // defined in link.h
 }
 
+uptr FindDynamicShadowStart() {
+  UNREACHABLE("FindDynamicShadowStart is not available");
+  return 0;
+}
+
 void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
   UNIMPLEMENTED();
 }
@@ -140,9 +145,9 @@ void AsanCheckIncompatibleRT() {
       // system libraries, causing crashes later in ASan initialization.
       MemoryMappingLayout proc_maps(/*cache_enabled*/true);
       char filename[128];
-      while (proc_maps.Next(nullptr, nullptr, nullptr, filename,
-                            sizeof(filename), nullptr)) {
-        if (IsDynamicRTName(filename)) {
+      MemoryMappedSegment segment(filename, sizeof(filename));
+      while (proc_maps.Next(&segment)) {
+        if (IsDynamicRTName(segment.filename)) {
           Report("Your application is linked against "
                  "incompatible ASan runtimes.\n");
           Die();
diff --git a/lib/asan/asan_mac.cc b/lib/asan/asan_mac.cc
index 3c93b26d9bf6..b7af1a58664c 100644
--- a/lib/asan/asan_mac.cc
+++ b/lib/asan/asan_mac.cc
@@ -55,6 +55,29 @@ void *AsanDoesNotSupportStaticLinkage() {
   return 0;
 }
 
+uptr FindDynamicShadowStart() {
+  uptr granularity = GetMmapGranularity();
+  uptr alignment = 8 * granularity;
+  uptr left_padding = granularity;
+  uptr space_size = kHighShadowEnd + left_padding;
+
+  uptr largest_gap_found = 0;
+  uptr shadow_start = FindAvailableMemoryRange(space_size, alignment,
+                                               granularity, &largest_gap_found);
+  // If the shadow doesn't fit, restrict the address space to make it fit.
+  if (shadow_start == 0) {
+    uptr new_max_vm = RoundDownTo(largest_gap_found << SHADOW_SCALE, alignment);
+    RestrictMemoryToMaxAddress(new_max_vm);
+    kHighMemEnd = new_max_vm - 1;
+    space_size = kHighShadowEnd + left_padding;
+    shadow_start =
+        FindAvailableMemoryRange(space_size, alignment, granularity, nullptr);
+  }
+  CHECK_NE((uptr)0, shadow_start);
+  CHECK(IsAligned(shadow_start, alignment));
+  return shadow_start;
+}
+
 // No-op. Mac does not support static linkage anyway.
 void AsanCheckDynamicRTPrereqs() {}
 
diff --git a/lib/asan/asan_new_delete.cc b/lib/asan/asan_new_delete.cc
index 942b169d920c..e68c7f3e2400 100644
--- a/lib/asan/asan_new_delete.cc
+++ b/lib/asan/asan_new_delete.cc
@@ -26,7 +26,7 @@
 // VS2015 dynamic CRT (MD) work.
 #if SANITIZER_WINDOWS
 #define CXX_OPERATOR_ATTRIBUTE
-#define COMMENT_EXPORT(sym) __pragma(comment(linker, "/export:"##sym))
+#define COMMENT_EXPORT(sym) __pragma(comment(linker, "/export:" sym))
 #ifdef _WIN64
 COMMENT_EXPORT("??2@YAPEAX_K@Z")                     // operator new
 COMMENT_EXPORT("??2@YAPEAX_KAEBUnothrow_t@std@@@Z")  // operator new nothrow
diff --git a/lib/asan/asan_posix.cc b/lib/asan/asan_posix.cc
index 68fde9139232..added746ace8 100644
--- a/lib/asan/asan_posix.cc
+++ b/lib/asan/asan_posix.cc
@@ -59,7 +59,7 @@ void AsanOnDeadlySignal(int signo, void *siginfo, void *context) {
   //   lis r0,-10000
   //   stdux r1,r1,r0 # store sp to [sp-10000] and update sp by -10000
   // If the store faults then sp will not have been updated, so test above
-  // will not work, becase the fault address will be more than just "slightly"
+  // will not work, because the fault address will be more than just "slightly"
   // below sp.
   if (!IsStackAccess && IsAccessibleMemoryRange(sig.pc, 4)) {
     u32 inst = *(unsigned *)sig.pc;
diff --git a/lib/asan/asan_rtl.cc b/lib/asan/asan_rtl.cc
index d9d7d7e4f13d..5ae3568ae04a 100644
--- a/lib/asan/asan_rtl.cc
+++ b/lib/asan/asan_rtl.cc
@@ -438,15 +438,7 @@ static void InitializeShadowMemory() {
   if (shadow_start == kDefaultShadowSentinel) {
     __asan_shadow_memory_dynamic_address = 0;
     CHECK_EQ(0, kLowShadowBeg);
-
-    uptr granularity = GetMmapGranularity();
-    uptr alignment = 8 * granularity;
-    uptr left_padding = granularity;
-    uptr space_size = kHighShadowEnd + left_padding;
-
-    shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity);
-    CHECK_NE((uptr)0, shadow_start);
-    CHECK(IsAligned(shadow_start, alignment));
+    shadow_start = FindDynamicShadowStart();
   }
   // Update the shadow memory address (potentially) used by instrumentation.
   __asan_shadow_memory_dynamic_address = shadow_start;
diff --git a/lib/asan/asan_thread.cc b/lib/asan/asan_thread.cc
index 714496d5c606..b1a0d9a3b37f 100644
--- a/lib/asan/asan_thread.cc
+++ b/lib/asan/asan_thread.cc
@@ -200,7 +200,6 @@ FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() {
   uptr stack_size = this->stack_size();
   if (stack_size == 0)  // stack_size is not yet available, don't use FakeStack.
     return nullptr;
-  CHECK_LE(stack_size, 0x10000000);
   uptr old_val = 0;
   // fake_stack_ has 3 states:
   // 0   -- not initialized
diff --git a/lib/asan/asan_win.cc b/lib/asan/asan_win.cc
index 26db32465da8..8a839d913f95 100644
--- a/lib/asan/asan_win.cc
+++ b/lib/asan/asan_win.cc
@@ -217,6 +217,18 @@ void *AsanDoesNotSupportStaticLinkage() {
   return 0;
 }
 
+uptr FindDynamicShadowStart() {
+  uptr granularity = GetMmapGranularity();
+  uptr alignment = 8 * granularity;
+  uptr left_padding = granularity;
+  uptr space_size = kHighShadowEnd + left_padding;
+  uptr shadow_start =
+      FindAvailableMemoryRange(space_size, alignment, granularity, nullptr);
+  CHECK_NE((uptr)0, shadow_start);
+  CHECK(IsAligned(shadow_start, alignment));
+  return shadow_start;
+}
+
 void AsanCheckDynamicRTPrereqs() {}
 
 void AsanCheckIncompatibleRT() {}
diff --git a/lib/asan/scripts/asan_device_setup b/lib/asan/scripts/asan_device_setup
index 79ac2f916569..5a4f7c47cc21 100755
--- a/lib/asan/scripts/asan_device_setup
+++ b/lib/asan/scripts/asan_device_setup
@@ -52,7 +52,7 @@ function adb_remount {
     local STORAGE=`$ADB shell mount | grep /system | cut -d ' ' -f1`
     if [ "$STORAGE" != "" ]; then
       echo Remounting $STORAGE at /system
-      $ADB shell su -c "mount -o remount,rw $STORAGE /system"
+      $ADB shell su -c "mount -o rw,remount $STORAGE /system"
     else
       echo Failed to get storage device name for "/system" mount point
     fi
diff --git a/lib/builtins/CMakeLists.txt b/lib/builtins/CMakeLists.txt
index 6556e7ac6513..f0d3f50714c1 100644
--- a/lib/builtins/CMakeLists.txt
+++ b/lib/builtins/CMakeLists.txt
@@ -44,7 +44,6 @@ set(GENERIC_SOURCES
   ashrti3.c
   bswapdi2.c
   bswapsi2.c
-  clear_cache.c
   clzdi2.c
   clzsi2.c
   clzti2.c
@@ -68,7 +67,6 @@ set(GENERIC_SOURCES
   divti3.c
   divtf3.c
   divxc3.c
-  eprintf.c
   extendsfdf2.c
   extendhfsf2.c
   ffsdi2.c
@@ -191,11 +189,12 @@ option(COMPILER_RT_EXCLUDE_ATOMIC_BUILTIN
   "Skip the atomic builtin (this may be needed if system headers are unavailable)"
   Off)
 
-if(NOT COMPILER_RT_BAREMETAL_BUILD)
+if(NOT FUCHSIA AND NOT COMPILER_RT_BAREMETAL_BUILD)
   set(GENERIC_SOURCES
     ${GENERIC_SOURCES}
     emutls.c 
-    enable_execute_stack.c)
+    enable_execute_stack.c
+    eprintf.c)
 endif()
 
 if(COMPILER_RT_HAS_ATOMIC_KEYWORD AND NOT COMPILER_RT_EXCLUDE_ATOMIC_BUILTIN)
@@ -221,6 +220,12 @@ if (HAVE_UNWIND_H)
       gcc_personality_v0.c)
 endif ()
 
+if (NOT FUCHSIA)
+  set(GENERIC_SOURCES
+    ${GENERIC_SOURCES}
+    clear_cache.c)
+endif()
+
 if (NOT MSVC)
   set(x86_64_SOURCES
       x86_64/chkstk.S
diff --git a/lib/builtins/cpu_model.c b/lib/builtins/cpu_model.c
index 5ff6baf43876..c6b30eda0a77 100644
--- a/lib/builtins/cpu_model.c
+++ b/lib/builtins/cpu_model.c
@@ -44,29 +44,16 @@ enum ProcessorVendors {
 };
 
 enum ProcessorTypes {
-  INTEL_ATOM = 1,
+  INTEL_BONNELL = 1,
   INTEL_CORE2,
   INTEL_COREI7,
   AMDFAM10H,
   AMDFAM15H,
-  INTEL_i386,
-  INTEL_i486,
-  INTEL_PENTIUM,
-  INTEL_PENTIUM_PRO,
-  INTEL_PENTIUM_II,
-  INTEL_PENTIUM_III,
-  INTEL_PENTIUM_IV,
-  INTEL_PENTIUM_M,
-  INTEL_CORE_DUO,
-  INTEL_XEONPHI,
-  INTEL_X86_64,
-  INTEL_NOCONA,
-  INTEL_PRESCOTT,
-  AMD_i486,
-  AMDPENTIUM,
-  AMDATHLON,
-  AMDFAM14H,
-  AMDFAM16H,
+  INTEL_SILVERMONT,
+  INTEL_KNL,
+  AMD_BTVER1,
+  AMD_BTVER2,
+  AMDFAM17H,
   CPU_TYPE_MAX
 };
 
@@ -79,32 +66,14 @@ enum ProcessorSubtypes {
   AMDFAM10H_ISTANBUL,
   AMDFAM15H_BDVER1,
   AMDFAM15H_BDVER2,
-  INTEL_PENTIUM_MMX,
-  INTEL_CORE2_65,
-  INTEL_CORE2_45,
+  AMDFAM15H_BDVER3,
+  AMDFAM15H_BDVER4,
+  AMDFAM17H_ZNVER1,
   INTEL_COREI7_IVYBRIDGE,
   INTEL_COREI7_HASWELL,
   INTEL_COREI7_BROADWELL,
   INTEL_COREI7_SKYLAKE,
   INTEL_COREI7_SKYLAKE_AVX512,
-  INTEL_ATOM_BONNELL,
-  INTEL_ATOM_SILVERMONT,
-  INTEL_KNIGHTS_LANDING,
-  AMDPENTIUM_K6,
-  AMDPENTIUM_K62,
-  AMDPENTIUM_K63,
-  AMDPENTIUM_GEODE,
-  AMDATHLON_TBIRD,
-  AMDATHLON_MP,
-  AMDATHLON_XP,
-  AMDATHLON_K8SSE3,
-  AMDATHLON_OPTERON,
-  AMDATHLON_FX,
-  AMDATHLON_64,
-  AMD_BTVER1,
-  AMD_BTVER2,
-  AMDFAM15H_BDVER3,
-  AMDFAM15H_BDVER4,
   CPU_SUBTYPE_MAX
 };
 
@@ -120,11 +89,26 @@ enum ProcessorFeatures {
   FEATURE_SSE4_2,
   FEATURE_AVX,
   FEATURE_AVX2,
-  FEATURE_AVX512,
-  FEATURE_AVX512SAVE,
-  FEATURE_MOVBE,
-  FEATURE_ADX,
-  FEATURE_EM64T
+  FEATURE_SSE4_A,
+  FEATURE_FMA4,
+  FEATURE_XOP,
+  FEATURE_FMA,
+  FEATURE_AVX512F,
+  FEATURE_BMI,
+  FEATURE_BMI2,
+  FEATURE_AES,
+  FEATURE_PCLMUL,
+  FEATURE_AVX512VL,
+  FEATURE_AVX512BW,
+  FEATURE_AVX512DQ,
+  FEATURE_AVX512CD,
+  FEATURE_AVX512ER,
+  FEATURE_AVX512PF,
+  FEATURE_AVX512VBMI,
+  FEATURE_AVX512IFMA,
+  FEATURE_AVX5124VNNIW,
+  FEATURE_AVX5124FMAPS,
+  FEATURE_AVX512VPOPCNTDQ
 };
 
 // The check below for i386 was copied from clang's cpuid.h (__get_cpuid_max).
@@ -164,26 +148,27 @@ static bool isCpuIdSupported() {
 
 /// getX86CpuIDAndInfo - Execute the specified cpuid and return the 4 values in
 /// the specified arguments.  If we can't run cpuid on the host, return true.
-static void getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX,
+static bool getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX,
                                unsigned *rECX, unsigned *rEDX) {
 #if defined(__GNUC__) || defined(__clang__)
 #if defined(__x86_64__)
-  // gcc doesn't know cpuid would clobber ebx/rbx. Preseve it manually.
+  // gcc doesn't know cpuid would clobber ebx/rbx. Preserve it manually.
+  // FIXME: should we save this for Clang?
   __asm__("movq\t%%rbx, %%rsi\n\t"
           "cpuid\n\t"
           "xchgq\t%%rbx, %%rsi\n\t"
           : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
           : "a"(value));
+  return false;
 #elif defined(__i386__)
   __asm__("movl\t%%ebx, %%esi\n\t"
           "cpuid\n\t"
           "xchgl\t%%ebx, %%esi\n\t"
           : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
           : "a"(value));
-// pedantic #else returns to appease -Wunreachable-code (so we don't generate
-// postprocessed code that looks like "return true; return false;")
+  return false;
 #else
-  assert(0 && "This method is defined only for x86.");
+  return true;
 #endif
 #elif defined(_MSC_VER)
   // The MSVC intrinsic is portable across x86 and x64.
@@ -193,15 +178,16 @@ static void getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX,
   *rEBX = registers[1];
   *rECX = registers[2];
   *rEDX = registers[3];
+  return false;
 #else
-  assert(0 && "This method is defined only for GNUC, Clang or MSVC.");
+  return true;
 #endif
 }
 
 /// getX86CpuIDAndInfoEx - Execute the specified cpuid with subleaf and return
 /// the 4 values in the specified arguments.  If we can't run cpuid on the host,
 /// return true.
-static void getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf,
+static bool getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf,
                                  unsigned *rEAX, unsigned *rEBX, unsigned *rECX,
                                  unsigned *rEDX) {
 #if defined(__x86_64__) || defined(_M_X64)
@@ -213,6 +199,7 @@ static void getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf,
           "xchgq\t%%rbx, %%rsi\n\t"
           : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
           : "a"(value), "c"(subleaf));
+  return false;
 #elif defined(_MSC_VER)
   int registers[4];
   __cpuidex(registers, value, subleaf);
@@ -220,8 +207,9 @@ static void getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf,
   *rEBX = registers[1];
   *rECX = registers[2];
   *rEDX = registers[3];
+  return false;
 #else
-  assert(0 && "This method is defined only for GNUC, Clang or MSVC.");
+  return true;
 #endif
 #elif defined(__i386__) || defined(_M_IX86)
 #if defined(__GNUC__) || defined(__clang__)
@@ -230,6 +218,7 @@ static void getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf,
           "xchgl\t%%ebx, %%esi\n\t"
           : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
           : "a"(value), "c"(subleaf));
+  return false;
 #elif defined(_MSC_VER)
   __asm {
       mov   eax,value
@@ -244,11 +233,12 @@ static void getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf,
       mov   esi,rEDX
       mov   dword ptr [esi],edx
   }
+  return false;
 #else
-  assert(0 && "This method is defined only for GNUC, Clang or MSVC.");
+  return true;
 #endif
 #else
-  assert(0 && "This method is defined only for x86.");
+  return true;
 #endif
 }
 
@@ -283,84 +273,15 @@ static void detectX86FamilyModel(unsigned EAX, unsigned *Family,
   }
 }
 
-static void getIntelProcessorTypeAndSubtype(unsigned int Family,
-                                            unsigned int Model,
-                                            unsigned int Brand_id,
-                                            unsigned int Features,
-                                            unsigned *Type, unsigned *Subtype) {
+static void
+getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
+                                unsigned Brand_id, unsigned Features,
+                                unsigned *Type, unsigned *Subtype) {
   if (Brand_id != 0)
     return;
   switch (Family) {
-  case 3:
-    *Type = INTEL_i386;
-    break;
-  case 4:
-    switch (Model) {
-    case 0: // Intel486 DX processors
-    case 1: // Intel486 DX processors
-    case 2: // Intel486 SX processors
-    case 3: // Intel487 processors, IntelDX2 OverDrive processors,
-            // IntelDX2 processors
-    case 4: // Intel486 SL processor
-    case 5: // IntelSX2 processors
-    case 7: // Write-Back Enhanced IntelDX2 processors
-    case 8: // IntelDX4 OverDrive processors, IntelDX4 processors
-    default:
-      *Type = INTEL_i486;
-      break;
-    }
-  case 5:
-    switch (Model) {
-    case 1: // Pentium OverDrive processor for Pentium processor (60, 66),
-            // Pentium processors (60, 66)
-    case 2: // Pentium OverDrive processor for Pentium processor (75, 90,
-            // 100, 120, 133), Pentium processors (75, 90, 100, 120, 133,
-            // 150, 166, 200)
-    case 3: // Pentium OverDrive processors for Intel486 processor-based
-            // systems
-      *Type = INTEL_PENTIUM;
-      break;
-    case 4: // Pentium OverDrive processor with MMX technology for Pentium
-            // processor (75, 90, 100, 120, 133), Pentium processor with
-            // MMX technology (166, 200)
-      *Type = INTEL_PENTIUM;
-      *Subtype = INTEL_PENTIUM_MMX;
-      break;
-    default:
-      *Type = INTEL_PENTIUM;
-      break;
-    }
   case 6:
     switch (Model) {
-    case 0x01: // Pentium Pro processor
-      *Type = INTEL_PENTIUM_PRO;
-      break;
-    case 0x03: // Intel Pentium II OverDrive processor, Pentium II processor,
-               // model 03
-    case 0x05: // Pentium II processor, model 05, Pentium II Xeon processor,
-               // model 05, and Intel Celeron processor, model 05
-    case 0x06: // Celeron processor, model 06
-      *Type = INTEL_PENTIUM_II;
-      break;
-    case 0x07: // Pentium III processor, model 07, and Pentium III Xeon
-               // processor, model 07
-    case 0x08: // Pentium III processor, model 08, Pentium III Xeon processor,
-               // model 08, and Celeron processor, model 08
-    case 0x0a: // Pentium III Xeon processor, model 0Ah
-    case 0x0b: // Pentium III processor, model 0Bh
-      *Type = INTEL_PENTIUM_III;
-      break;
-    case 0x09: // Intel Pentium M processor, Intel Celeron M processor model 09.
-    case 0x0d: // Intel Pentium M processor, Intel Celeron M processor, model
-               // 0Dh. All processors are manufactured using the 90 nm process.
-    case 0x15: // Intel EP80579 Integrated Processor and Intel EP80579
-               // Integrated Processor with Intel QuickAssist Technology
-      *Type = INTEL_PENTIUM_M;
-      break;
-    case 0x0e: // Intel Core Duo processor, Intel Core Solo processor, model
-               // 0Eh. All processors are manufactured using the 65 nm process.
-      *Type = INTEL_CORE_DUO;
-      break;   // yonah
     case 0x0f: // Intel Core 2 Duo processor, Intel Core 2 Duo mobile
                // processor, Intel Core 2 Quad processor, Intel Core 2 Quad
                // mobile processor, Intel Core 2 Extreme processor, Intel
@@ -368,9 +289,6 @@ static void getIntelProcessorTypeAndSubtype(unsigned int Family,
                // 0Fh. All processors are manufactured using the 65 nm process.
     case 0x16: // Intel Celeron processor model 16h. All processors are
                // manufactured using the 65 nm process
-      *Type = INTEL_CORE2; // "core2"
-      *Subtype = INTEL_CORE2_65;
-      break;
     case 0x17: // Intel Core 2 Extreme processor, Intel Xeon processor, model
                // 17h. All processors are manufactured using the 45 nm process.
                //
@@ -378,14 +296,13 @@ static void getIntelProcessorTypeAndSubtype(unsigned int Family,
     case 0x1d: // Intel Xeon processor MP. All processors are manufactured using
                // the 45 nm process.
       *Type = INTEL_CORE2; // "penryn"
-      *Subtype = INTEL_CORE2_45;
       break;
     case 0x1a: // Intel Core i7 processor and Intel Xeon processor. All
                // processors are manufactured using the 45 nm process.
     case 0x1e: // Intel(R) Core(TM) i7 CPU         870  @ 2.93GHz.
                // As found in a Summer 2010 model iMac.
     case 0x1f:
-    case 0x2e:              // Nehalem EX
+    case 0x2e:             // Nehalem EX
       *Type = INTEL_COREI7; // "nehalem"
       *Subtype = INTEL_COREI7_NEHALEM;
       break;
@@ -403,7 +320,7 @@ static void getIntelProcessorTypeAndSubtype(unsigned int Family,
       *Subtype = INTEL_COREI7_SANDYBRIDGE;
       break;
     case 0x3a:
-    case 0x3e:              // Ivy Bridge EP
+    case 0x3e:             // Ivy Bridge EP
       *Type = INTEL_COREI7; // "ivybridge"
       *Subtype = INTEL_COREI7_IVYBRIDGE;
       break;
@@ -427,22 +344,26 @@ static void getIntelProcessorTypeAndSubtype(unsigned int Family,
       break;
 
     // Skylake:
-    case 0x4e:
-      *Type = INTEL_COREI7; // "skylake-avx512"
-      *Subtype = INTEL_COREI7_SKYLAKE_AVX512;
-      break;
-    case 0x5e:
+    case 0x4e: // Skylake mobile
+    case 0x5e: // Skylake desktop
+    case 0x8e: // Kaby Lake mobile
+    case 0x9e: // Kaby Lake desktop
       *Type = INTEL_COREI7; // "skylake"
       *Subtype = INTEL_COREI7_SKYLAKE;
       break;
 
+    // Skylake Xeon:
+    case 0x55:
+      *Type = INTEL_COREI7;
+      *Subtype = INTEL_COREI7_SKYLAKE_AVX512; // "skylake-avx512"
+      break;
+
     case 0x1c: // Most 45 nm Intel Atom processors
     case 0x26: // 45 nm Atom Lincroft
     case 0x27: // 32 nm Atom Medfield
     case 0x35: // 32 nm Atom Midview
     case 0x36: // 32 nm Atom Midview
-      *Type = INTEL_ATOM;
-      *Subtype = INTEL_ATOM_BONNELL;
+      *Type = INTEL_BONNELL;
       break; // "bonnell"
 
     // Atom Silvermont codes from the Intel software optimization guide.
@@ -452,185 +373,29 @@ static void getIntelProcessorTypeAndSubtype(unsigned int Family,
     case 0x5a:
     case 0x5d:
     case 0x4c: // really airmont
-      *Type = INTEL_ATOM;
-      *Subtype = INTEL_ATOM_SILVERMONT;
+      *Type = INTEL_SILVERMONT;
       break; // "silvermont"
 
     case 0x57:
-      *Type = INTEL_XEONPHI; // knl
-      *Subtype = INTEL_KNIGHTS_LANDING;
-      break;
-
-    default: // Unknown family 6 CPU, try to guess.
-      if (Features & (1 << FEATURE_AVX512)) {
-        *Type = INTEL_XEONPHI; // knl
-        *Subtype = INTEL_KNIGHTS_LANDING;
-        break;
-      }
-      if (Features & (1 << FEATURE_ADX)) {
-        *Type = INTEL_COREI7;
-        *Subtype = INTEL_COREI7_BROADWELL;
-        break;
-      }
-      if (Features & (1 << FEATURE_AVX2)) {
-        *Type = INTEL_COREI7;
-        *Subtype = INTEL_COREI7_HASWELL;
-        break;
-      }
-      if (Features & (1 << FEATURE_AVX)) {
-        *Type = INTEL_COREI7;
-        *Subtype = INTEL_COREI7_SANDYBRIDGE;
-        break;
-      }
-      if (Features & (1 << FEATURE_SSE4_2)) {
-        if (Features & (1 << FEATURE_MOVBE)) {
-          *Type = INTEL_ATOM;
-          *Subtype = INTEL_ATOM_SILVERMONT;
-        } else {
-          *Type = INTEL_COREI7;
-          *Subtype = INTEL_COREI7_NEHALEM;
-        }
-        break;
-      }
-      if (Features & (1 << FEATURE_SSE4_1)) {
-        *Type = INTEL_CORE2; // "penryn"
-        *Subtype = INTEL_CORE2_45;
-        break;
-      }
-      if (Features & (1 << FEATURE_SSSE3)) {
-        if (Features & (1 << FEATURE_MOVBE)) {
-          *Type = INTEL_ATOM;
-          *Subtype = INTEL_ATOM_BONNELL; // "bonnell"
-        } else {
-          *Type = INTEL_CORE2; // "core2"
-          *Subtype = INTEL_CORE2_65;
-        }
-        break;
-      }
-      if (Features & (1 << FEATURE_EM64T)) {
-        *Type = INTEL_X86_64;
-        break; // x86-64
-      }
-      if (Features & (1 << FEATURE_SSE2)) {
-        *Type = INTEL_PENTIUM_M;
-        break;
-      }
-      if (Features & (1 << FEATURE_SSE)) {
-        *Type = INTEL_PENTIUM_III;
-        break;
-      }
-      if (Features & (1 << FEATURE_MMX)) {
-        *Type = INTEL_PENTIUM_II;
-        break;
-      }
-      *Type = INTEL_PENTIUM_PRO;
-      break;
-    }
-  case 15: {
-    switch (Model) {
-    case 0: // Pentium 4 processor, Intel Xeon processor. All processors are
-            // model 00h and manufactured using the 0.18 micron process.
-    case 1: // Pentium 4 processor, Intel Xeon processor, Intel Xeon
-            // processor MP, and Intel Celeron processor. All processors are
-            // model 01h and manufactured using the 0.18 micron process.
-    case 2: // Pentium 4 processor, Mobile Intel Pentium 4 processor - M,
-            // Intel Xeon processor, Intel Xeon processor MP, Intel Celeron
-            // processor, and Mobile Intel Celeron processor. All processors
-            // are model 02h and manufactured using the 0.13 micron process.
-      *Type =
-          ((Features & (1 << FEATURE_EM64T)) ? INTEL_X86_64 : INTEL_PENTIUM_IV);
+      *Type = INTEL_KNL; // knl
       break;
 
-    case 3: // Pentium 4 processor, Intel Xeon processor, Intel Celeron D
-            // processor. All processors are model 03h and manufactured using
-            // the 90 nm process.
-    case 4: // Pentium 4 processor, Pentium 4 processor Extreme Edition,
-            // Pentium D processor, Intel Xeon processor, Intel Xeon
-            // processor MP, Intel Celeron D processor. All processors are
-            // model 04h and manufactured using the 90 nm process.
-    case 6: // Pentium 4 processor, Pentium D processor, Pentium processor
-            // Extreme Edition, Intel Xeon processor, Intel Xeon processor
-            // MP, Intel Celeron D processor. All processors are model 06h
-            // and manufactured using the 65 nm process.
-      *Type =
-          ((Features & (1 << FEATURE_EM64T)) ? INTEL_NOCONA : INTEL_PRESCOTT);
-      break;
-
-    default:
-      *Type =
-          ((Features & (1 << FEATURE_EM64T)) ? INTEL_X86_64 : INTEL_PENTIUM_IV);
+    default: // Unknown family 6 CPU.
       break;
+    break;
     }
-  }
   default:
-    break; /*"generic"*/
+    break; // Unknown.
   }
 }
 
-static void getAMDProcessorTypeAndSubtype(unsigned int Family,
-                                          unsigned int Model,
-                                          unsigned int Features, unsigned *Type,
+static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
+                                          unsigned Features, unsigned *Type,
                                           unsigned *Subtype) {
   // FIXME: this poorly matches the generated SubtargetFeatureKV table.  There
   // appears to be no way to generate the wide variety of AMD-specific targets
   // from the information returned from CPUID.
   switch (Family) {
-  case 4:
-    *Type = AMD_i486;
-  case 5:
-    *Type = AMDPENTIUM;
-    switch (Model) {
-    case 6:
-    case 7:
-      *Subtype = AMDPENTIUM_K6;
-      break; // "k6"
-    case 8:
-      *Subtype = AMDPENTIUM_K62;
-      break; // "k6-2"
-    case 9:
-    case 13:
-      *Subtype = AMDPENTIUM_K63;
-      break; // "k6-3"
-    case 10:
-      *Subtype = AMDPENTIUM_GEODE;
-      break; // "geode"
-    default:
-      break;
-    }
-  case 6:
-    *Type = AMDATHLON;
-    switch (Model) {
-    case 4:
-      *Subtype = AMDATHLON_TBIRD;
-      break; // "athlon-tbird"
-    case 6:
-    case 7:
-    case 8:
-      *Subtype = AMDATHLON_MP;
-      break; // "athlon-mp"
-    case 10:
-      *Subtype = AMDATHLON_XP;
-      break; // "athlon-xp"
-    default:
-      break;
-    }
-  case 15:
-    *Type = AMDATHLON;
-    if (Features & (1 << FEATURE_SSE3)) {
-      *Subtype = AMDATHLON_K8SSE3;
-      break; // "k8-sse3"
-    }
-    switch (Model) {
-    case 1:
-      *Subtype = AMDATHLON_OPTERON;
-      break; // "opteron"
-    case 5:
-      *Subtype = AMDATHLON_FX;
-      break; // "athlon-fx"; also opteron
-    default:
-      *Subtype = AMDATHLON_64;
-      break; // "athlon64"
-    }
   case 16:
     *Type = AMDFAM10H; // "amdfam10"
     switch (Model) {
@@ -643,23 +408,16 @@ static void getAMDProcessorTypeAndSubtype(unsigned int Family,
     case 8:
       *Subtype = AMDFAM10H_ISTANBUL;
       break;
-    default:
-      break;
     }
+    break;
   case 20:
-    *Type = AMDFAM14H;
-    *Subtype = AMD_BTVER1;
+    *Type = AMD_BTVER1;
     break; // "btver1";
   case 21:
     *Type = AMDFAM15H;
-    if (!(Features &
-          (1 << FEATURE_AVX))) { // If no AVX support, provide a sane fallback.
-      *Subtype = AMD_BTVER1;
-      break; // "btver1"
-    }
-    if (Model >= 0x50 && Model <= 0x6f) {
+    if (Model >= 0x60 && Model <= 0x7f) {
       *Subtype = AMDFAM15H_BDVER4;
-      break; // "bdver4"; 50h-6Fh: Excavator
+      break; // "bdver4"; 60h-7Fh: Excavator
     }
     if (Model >= 0x30 && Model <= 0x3f) {
       *Subtype = AMDFAM15H_BDVER3;
@@ -675,31 +433,47 @@ static void getAMDProcessorTypeAndSubtype(unsigned int Family,
     }
     break;
   case 22:
-    *Type = AMDFAM16H;
-    if (!(Features &
-          (1 << FEATURE_AVX))) { // If no AVX support provide a sane fallback.
-      *Subtype = AMD_BTVER1;
-      break; // "btver1";
-    }
-    *Subtype = AMD_BTVER2;
+    *Type = AMD_BTVER2;
     break; // "btver2"
+  case 23:
+    *Type = AMDFAM17H;
+    *Subtype = AMDFAM17H_ZNVER1;
+    break;
   default:
     break; // "generic"
   }
 }
 
-static unsigned getAvailableFeatures(unsigned int ECX, unsigned int EDX,
-                                     unsigned MaxLeaf) {
+static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
+                                 unsigned *FeaturesOut) {
   unsigned Features = 0;
-  unsigned int EAX, EBX;
-  Features |= (((EDX >> 23) & 1) << FEATURE_MMX);
-  Features |= (((EDX >> 25) & 1) << FEATURE_SSE);
-  Features |= (((EDX >> 26) & 1) << FEATURE_SSE2);
-  Features |= (((ECX >> 0) & 1) << FEATURE_SSE3);
-  Features |= (((ECX >> 9) & 1) << FEATURE_SSSE3);
-  Features |= (((ECX >> 19) & 1) << FEATURE_SSE4_1);
-  Features |= (((ECX >> 20) & 1) << FEATURE_SSE4_2);
-  Features |= (((ECX >> 22) & 1) << FEATURE_MOVBE);
+  unsigned EAX, EBX;
+
+  if ((EDX >> 15) & 1)
+    Features |= 1 << FEATURE_CMOV;
+  if ((EDX >> 23) & 1)
+    Features |= 1 << FEATURE_MMX;
+  if ((EDX >> 25) & 1)
+    Features |= 1 << FEATURE_SSE;
+  if ((EDX >> 26) & 1)
+    Features |= 1 << FEATURE_SSE2;
+
+  if ((ECX >> 0) & 1)
+    Features |= 1 << FEATURE_SSE3;
+  if ((ECX >> 1) & 1)
+    Features |= 1 << FEATURE_PCLMUL;
+  if ((ECX >> 9) & 1)
+    Features |= 1 << FEATURE_SSSE3;
+  if ((ECX >> 12) & 1)
+    Features |= 1 << FEATURE_FMA;
+  if ((ECX >> 19) & 1)
+    Features |= 1 << FEATURE_SSE4_1;
+  if ((ECX >> 20) & 1)
+    Features |= 1 << FEATURE_SSE4_2;
+  if ((ECX >> 23) & 1)
+    Features |= 1 << FEATURE_POPCNT;
+  if ((ECX >> 25) & 1)
+    Features |= 1 << FEATURE_AES;
 
   // If CPUID indicates support for XSAVE, XRESTORE and AVX, and XGETBV
   // indicates that the AVX registers will be saved and restored on context
@@ -708,20 +482,59 @@ static unsigned getAvailableFeatures(unsigned int ECX, unsigned int EDX,
   bool HasAVX = ((ECX & AVXBits) == AVXBits) && !getX86XCR0(&EAX, &EDX) &&
                 ((EAX & 0x6) == 0x6);
   bool HasAVX512Save = HasAVX && ((EAX & 0xe0) == 0xe0);
-  bool HasLeaf7 = MaxLeaf >= 0x7;
-  getX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX);
-  bool HasADX = HasLeaf7 && ((EBX >> 19) & 1);
-  bool HasAVX2 = HasAVX && HasLeaf7 && (EBX & 0x20);
-  bool HasAVX512 = HasLeaf7 && HasAVX512Save && ((EBX >> 16) & 1);
-  Features |= (HasAVX << FEATURE_AVX);
-  Features |= (HasAVX2 << FEATURE_AVX2);
-  Features |= (HasAVX512 << FEATURE_AVX512);
-  Features |= (HasAVX512Save << FEATURE_AVX512SAVE);
-  Features |= (HasADX << FEATURE_ADX);
-
-  getX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX);
-  Features |= (((EDX >> 29) & 0x1) << FEATURE_EM64T);
-  return Features;
+
+  if (HasAVX)
+    Features |= 1 << FEATURE_AVX;
+
+  bool HasLeaf7 =
+      MaxLeaf >= 0x7 && !getX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX);
+
+  if (HasLeaf7 && ((EBX >> 3) & 1))
+    Features |= 1 << FEATURE_BMI;
+  if (HasLeaf7 && ((EBX >> 5) & 1) && HasAVX)
+    Features |= 1 << FEATURE_AVX2;
+  if (HasLeaf7 && ((EBX >> 9) & 1))
+    Features |= 1 << FEATURE_BMI2;
+  if (HasLeaf7 && ((EBX >> 16) & 1) && HasAVX512Save)
+    Features |= 1 << FEATURE_AVX512F;
+  if (HasLeaf7 && ((EBX >> 17) & 1) && HasAVX512Save)
+    Features |= 1 << FEATURE_AVX512DQ;
+  if (HasLeaf7 && ((EBX >> 21) & 1) && HasAVX512Save)
+    Features |= 1 << FEATURE_AVX512IFMA;
+  if (HasLeaf7 && ((EBX >> 26) & 1) && HasAVX512Save)
+    Features |= 1 << FEATURE_AVX512PF;
+  if (HasLeaf7 && ((EBX >> 27) & 1) && HasAVX512Save)
+    Features |= 1 << FEATURE_AVX512ER;
+  if (HasLeaf7 && ((EBX >> 28) & 1) && HasAVX512Save)
+    Features |= 1 << FEATURE_AVX512CD;
+  if (HasLeaf7 && ((EBX >> 30) & 1) && HasAVX512Save)
+    Features |= 1 << FEATURE_AVX512BW;
+  if (HasLeaf7 && ((EBX >> 31) & 1) && HasAVX512Save)
+    Features |= 1 << FEATURE_AVX512VL;
+
+  if (HasLeaf7 && ((ECX >> 1) & 1) && HasAVX512Save)
+    Features |= 1 << FEATURE_AVX512VBMI;
+  if (HasLeaf7 && ((ECX >> 14) & 1) && HasAVX512Save)
+    Features |= 1 << FEATURE_AVX512VPOPCNTDQ;
+
+  if (HasLeaf7 && ((EDX >> 2) & 1) && HasAVX512Save)
+    Features |= 1 << FEATURE_AVX5124VNNIW;
+  if (HasLeaf7 && ((EDX >> 3) & 1) && HasAVX512Save)
+    Features |= 1 << FEATURE_AVX5124FMAPS;
+
+  unsigned MaxExtLevel;
+  getX86CpuIDAndInfo(0x80000000, &MaxExtLevel, &EBX, &ECX, &EDX);
+
+  bool HasExtLeaf1 = MaxExtLevel >= 0x80000001 &&
+                     !getX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX);
+  if (HasExtLeaf1 && ((ECX >> 6) & 1))
+    Features |= 1 << FEATURE_SSE4_A;
+  if (HasExtLeaf1 && ((ECX >> 11) & 1))
+    Features |= 1 << FEATURE_XOP;
+  if (HasExtLeaf1 && ((ECX >> 16) & 1))
+    Features |= 1 << FEATURE_FMA4;
+
+  *FeaturesOut = Features;
 }
 
 #if defined(HAVE_INIT_PRIORITY)
@@ -751,11 +564,11 @@ struct __processor_model {
 
 int CONSTRUCTOR_ATTRIBUTE
 __cpu_indicator_init(void) {
-  unsigned int EAX, EBX, ECX, EDX;
-  unsigned int MaxLeaf = 5;
-  unsigned int Vendor;
-  unsigned int Model, Family, Brand_id;
-  unsigned int Features = 0;
+  unsigned EAX, EBX, ECX, EDX;
+  unsigned MaxLeaf = 5;
+  unsigned Vendor;
+  unsigned Model, Family, Brand_id;
+  unsigned Features = 0;
 
   /* This function needs to run just once.  */
   if (__cpu_model.__cpu_vendor)
@@ -765,9 +578,7 @@ __cpu_indicator_init(void) {
     return -1;
 
   /* Assume cpuid insn present. Run in level 0 to get vendor id. */
-  getX86CpuIDAndInfo(0, &MaxLeaf, &Vendor, &ECX, &EDX);
-
-  if (MaxLeaf < 1) {
+  if (getX86CpuIDAndInfo(0, &MaxLeaf, &Vendor, &ECX, &EDX) || MaxLeaf < 1) {
     __cpu_model.__cpu_vendor = VENDOR_OTHER;
     return -1;
   }
@@ -776,7 +587,7 @@ __cpu_indicator_init(void) {
   Brand_id = EBX & 0xff;
 
   /* Find available features. */
-  Features = getAvailableFeatures(ECX, EDX, MaxLeaf);
+  getAvailableFeatures(ECX, EDX, MaxLeaf, &Features);
   __cpu_model.__cpu_features[0] = Features;
 
   if (Vendor == SIG_INTEL) {
diff --git a/lib/builtins/int_util.c b/lib/builtins/int_util.c
index 420d1e237aae..de87410dbca2 100644
--- a/lib/builtins/int_util.c
+++ b/lib/builtins/int_util.c
@@ -45,6 +45,16 @@ void compilerrt_abort_impl(const char *file, int line, const char *function) {
   __assert_rtn(function, file, line, "libcompiler_rt abort");
 }
 
+#elif __Fuchsia__
+
+#ifndef _WIN32
+__attribute__((weak))
+__attribute__((visibility("hidden")))
+#endif
+void compilerrt_abort_impl(const char *file, int line, const char *function) {
+  __builtin_trap();
+}
+
 #else
 
 /* Get the system definition of abort() */
diff --git a/lib/esan/working_set.cpp b/lib/esan/working_set.cpp
index f39111993c33..e56902c8f32a 100644
--- a/lib/esan/working_set.cpp
+++ b/lib/esan/working_set.cpp
@@ -160,15 +160,16 @@ static u32 countAndClearShadowValues(u32 BitIdx, uptr ShadowStart,
 static u32 computeWorkingSizeAndReset(u32 BitIdx) {
   u32 WorkingSetSize = 0;
   MemoryMappingLayout MemIter(true/*cache*/);
-  uptr Start, End, Prot;
-  while (MemIter.Next(&Start, &End, nullptr/*offs*/, nullptr/*file*/,
-                      0/*file size*/, &Prot)) {
-    VPrintf(4, "%s: considering %p-%p app=%d shadow=%d prot=%u\n",
-            __FUNCTION__, Start, End, Prot, isAppMem(Start),
-            isShadowMem(Start));
-    if (isShadowMem(Start) && (Prot & MemoryMappingLayout::kProtectionWrite)) {
-      VPrintf(3, "%s: walking %p-%p\n", __FUNCTION__, Start, End);
-      WorkingSetSize += countAndClearShadowValues(BitIdx, Start, End);
+  MemoryMappedSegment Segment;
+  while (MemIter.Next(&Segment)) {
+    VPrintf(4, "%s: considering %p-%p app=%d shadow=%d prot=%u\n", __FUNCTION__,
+            Segment.start, Segment.end, Segment.protection,
+            isAppMem(Segment.start), isShadowMem(Segment.start));
+    if (isShadowMem(Segment.start) && Segment.IsWritable()) {
+      VPrintf(3, "%s: walking %p-%p\n", __FUNCTION__, Segment.start,
+              Segment.end);
+      WorkingSetSize +=
+          countAndClearShadowValues(BitIdx, Segment.start, Segment.end);
     }
   }
   return WorkingSetSize;
diff --git a/lib/lsan/lsan_common.cc b/lib/lsan/lsan_common.cc
index a5ffc6835f5f..4ffa91568cc8 100644
--- a/lib/lsan/lsan_common.cc
+++ b/lib/lsan/lsan_common.cc
@@ -74,6 +74,10 @@ static const char kStdSuppressions[] =
   // definition.
   "leak:*pthread_exit*\n"
 #endif  // SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
+#if SANITIZER_MAC
+  // For Darwin and os_log/os_trace: https://reviews.llvm.org/D35173
+  "leak:*_os_trace*\n"
+#endif
   // TLS leak in some glibc versions, described in
   // https://sourceware.org/bugzilla/show_bug.cgi?id=12650.
   "leak:*tls_get_addr*\n";
@@ -301,11 +305,10 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
 }
 
 void ScanRootRegion(Frontier *frontier, const RootRegion &root_region,
-                    uptr region_begin, uptr region_end, uptr prot) {
+                    uptr region_begin, uptr region_end, bool is_readable) {
   uptr intersection_begin = Max(root_region.begin, region_begin);
   uptr intersection_end = Min(region_end, root_region.begin + root_region.size);
   if (intersection_begin >= intersection_end) return;
-  bool is_readable = prot & MemoryMappingLayout::kProtectionRead;
   LOG_POINTERS("Root region %p-%p intersects with mapped region %p-%p (%s)\n",
                root_region.begin, root_region.begin + root_region.size,
                region_begin, region_end,
@@ -318,11 +321,10 @@ void ScanRootRegion(Frontier *frontier, const RootRegion &root_region,
 static void ProcessRootRegion(Frontier *frontier,
                               const RootRegion &root_region) {
   MemoryMappingLayout proc_maps(/*cache_enabled*/ true);
-  uptr begin, end, prot;
-  while (proc_maps.Next(&begin, &end,
-                        /*offset*/ nullptr, /*filename*/ nullptr,
-                        /*filename_size*/ 0, &prot)) {
-    ScanRootRegion(frontier, root_region, begin, end, prot);
+  MemoryMappedSegment segment;
+  while (proc_maps.Next(&segment)) {
+    ScanRootRegion(frontier, root_region, segment.start, segment.end,
+                   segment.IsReadable());
   }
 }
 
diff --git a/lib/lsan/lsan_common.h b/lib/lsan/lsan_common.h
index beb31d6f40e4..d93ac1b10919 100644
--- a/lib/lsan/lsan_common.h
+++ b/lib/lsan/lsan_common.h
@@ -127,7 +127,7 @@ struct RootRegion {
 
 InternalMmapVector<RootRegion> const *GetRootRegions();
 void ScanRootRegion(Frontier *frontier, RootRegion const &region,
-                    uptr region_begin, uptr region_end, uptr prot);
+                    uptr region_begin, uptr region_end, bool is_readable);
 // Run stoptheworld while holding any platform-specific locks.
 void DoStopTheWorld(StopTheWorldCallback callback, void* argument);
 
diff --git a/lib/lsan/lsan_common_mac.cc b/lib/lsan/lsan_common_mac.cc
index adde3a1b4035..f87c6b7e0425 100644
--- a/lib/lsan/lsan_common_mac.cc
+++ b/lib/lsan/lsan_common_mac.cc
@@ -156,7 +156,7 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) {
     if (flags()->use_root_regions) {
       for (uptr i = 0; i < root_regions->size(); i++) {
         ScanRootRegion(frontier, (*root_regions)[i], address, end_address,
-                       info.protection);
+                       info.protection & kProtectionRead);
       }
     }
 
diff --git a/lib/msan/msan_interceptors.cc b/lib/msan/msan_interceptors.cc
index ce8444a3bb2f..069777c7f5e7 100644
--- a/lib/msan/msan_interceptors.cc
+++ b/lib/msan/msan_interceptors.cc
@@ -27,6 +27,7 @@
 #include "sanitizer_common/sanitizer_allocator_internal.h"
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_linux.h"
@@ -48,15 +49,9 @@ DECLARE_REAL(SIZE_T, strnlen, const char *s, SIZE_T maxlen)
 DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n)
 DECLARE_REAL(void *, memset, void *dest, int c, uptr n)
 
-#if SANITIZER_FREEBSD
-#define __errno_location __error
-#endif
-
 // True if this is a nested interceptor.
 static THREADLOCAL int in_interceptor_scope;
 
-extern "C" int *__errno_location(void);
-
 struct InterceptorScope {
   InterceptorScope() { ++in_interceptor_scope; }
   ~InterceptorScope() { --in_interceptor_scope; }
@@ -915,7 +910,7 @@ INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags,
   ENSURE_MSAN_INITED();
   if (addr && !MEM_IS_APP(addr)) {
     if (flags & map_fixed) {
-      *__errno_location() = errno_EINVAL;
+      errno = errno_EINVAL;
       return (void *)-1;
     } else {
       addr = nullptr;
@@ -933,7 +928,7 @@ INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags,
   ENSURE_MSAN_INITED();
   if (addr && !MEM_IS_APP(addr)) {
     if (flags & map_fixed) {
-      *__errno_location() = errno_EINVAL;
+      errno = errno_EINVAL;
       return (void *)-1;
     } else {
       addr = nullptr;
diff --git a/lib/sanitizer_common/CMakeLists.txt b/lib/sanitizer_common/CMakeLists.txt
index bf8459ef5e91..a17bd1299a2c 100644
--- a/lib/sanitizer_common/CMakeLists.txt
+++ b/lib/sanitizer_common/CMakeLists.txt
@@ -6,6 +6,7 @@ set(SANITIZER_SOURCES_NOTERMINATION
   sanitizer_common.cc
   sanitizer_deadlock_detector1.cc
   sanitizer_deadlock_detector2.cc
+  sanitizer_errno.cc
   sanitizer_flags.cc
   sanitizer_flag_parser.cc
   sanitizer_libc.cc
@@ -57,6 +58,7 @@ set(SANITIZER_LIBCDEP_SOURCES
   sanitizer_coverage_libcdep_new.cc
   sanitizer_coverage_win_sections.cc
   sanitizer_linux_libcdep.cc
+  sanitizer_mac_libcdep.cc
   sanitizer_posix_libcdep.cc
   sanitizer_stacktrace_libcdep.cc
   sanitizer_stoptheworld_linux_libcdep.cc
@@ -92,6 +94,8 @@ set(SANITIZER_HEADERS
   sanitizer_common_syscalls.inc
   sanitizer_deadlock_detector.h
   sanitizer_deadlock_detector_interface.h
+  sanitizer_errno.h
+  sanitizer_errno_codes.h
   sanitizer_flag_parser.h
   sanitizer_flags.h
   sanitizer_flags.inc
diff --git a/lib/sanitizer_common/sanitizer_common.h b/lib/sanitizer_common/sanitizer_common.h
index d44c71513896..89aae579856a 100644
--- a/lib/sanitizer_common/sanitizer_common.h
+++ b/lib/sanitizer_common/sanitizer_common.h
@@ -107,7 +107,8 @@ bool MprotectNoAccess(uptr addr, uptr size);
 bool MprotectReadOnly(uptr addr, uptr size);
 
 // Find an available address space.
-uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding);
+uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
+                              uptr *largest_gap_found);
 
 // Used to check if we can map shadow memory to a fixed location.
 bool MemoryRangeIsAvailable(uptr range_start, uptr range_end);
diff --git a/lib/sanitizer_common/sanitizer_common_interceptors.inc b/lib/sanitizer_common/sanitizer_common_interceptors.inc
index 459530aa95ba..8607bf44902d 100644
--- a/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -40,6 +40,7 @@
 
 #include "interception/interception.h"
 #include "sanitizer_addrhashmap.h"
+#include "sanitizer_errno.h"
 #include "sanitizer_placement_new.h"
 #include "sanitizer_platform_interceptors.h"
 #include "sanitizer_tls_get_addr.h"
diff --git a/lib/sanitizer_common/sanitizer_errno.cc b/lib/sanitizer_common/sanitizer_errno.cc
new file mode 100644
index 000000000000..a6f9fc6125eb
--- /dev/null
+++ b/lib/sanitizer_common/sanitizer_errno.cc
@@ -0,0 +1,35 @@
+//===-- sanitizer_errno.cc --------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizers run-time libraries.
+//
+// Defines errno to avoid including errno.h and its dependencies into other
+// files (e.g. interceptors are not supposed to include any system headers).
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_errno_codes.h"
+#include "sanitizer_internal_defs.h"
+
+#include <errno.h>
+
+namespace __sanitizer {
+
+COMPILER_CHECK(errno_ENOMEM == ENOMEM);
+COMPILER_CHECK(errno_EBUSY == EBUSY);
+COMPILER_CHECK(errno_EINVAL == EINVAL);
+
+// EOWNERDEAD is not present in some older platforms.
+#if defined(EOWNERDEAD)
+extern const int errno_EOWNERDEAD = EOWNERDEAD;
+#else
+extern const int errno_EOWNERDEAD = -1;
+#endif
+
+}  // namespace __sanitizer
diff --git a/lib/sanitizer_common/sanitizer_errno.h b/lib/sanitizer_common/sanitizer_errno.h
new file mode 100644
index 000000000000..c405307ba8ec
--- /dev/null
+++ b/lib/sanitizer_common/sanitizer_errno.h
@@ -0,0 +1,35 @@
+//===-- sanitizer_errno.h ---------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizers run-time libraries.
+//
+// Defines errno to avoid including errno.h and its dependencies into sensitive
+// files (e.g. interceptors are not supposed to include any system headers).
+// It's ok to use errno.h directly when your file already depend on other system
+// includes though.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ERRNO_H
+#define SANITIZER_ERRNO_H
+
+#include "sanitizer_errno_codes.h"
+#include "sanitizer_platform.h"
+
+#if SANITIZER_FREEBSD || SANITIZER_MAC
+#  define __errno_location __error
+#elif SANITIZER_ANDROID
+#  define __errno_location __errno
+#endif
+
+extern "C" int *__errno_location();
+
+#define errno (*__errno_location())
+
+#endif  // SANITIZER_ERRNO_H
diff --git a/lib/sanitizer_common/sanitizer_errno_codes.h b/lib/sanitizer_common/sanitizer_errno_codes.h
new file mode 100644
index 000000000000..dba774c5b6c5
--- /dev/null
+++ b/lib/sanitizer_common/sanitizer_errno_codes.h
@@ -0,0 +1,34 @@
+//===-- sanitizer_errno_codes.h ---------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizers run-time libraries.
+//
+// Defines errno codes to avoid including errno.h and its dependencies into
+// sensitive files (e.g. interceptors are not supposed to include any system
+// headers).
+// It's ok to use errno.h directly when your file already depend on other system
+// includes though.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ERRNO_CODES_H
+#define SANITIZER_ERRNO_CODES_H
+
+namespace __sanitizer {
+
+#define errno_ENOMEM 12
+#define errno_EBUSY 16
+#define errno_EINVAL 22
+
+// Those might not present or their value differ on different platforms.
+extern const int errno_EOWNERDEAD;
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_ERRNO_CODES_H
diff --git a/lib/sanitizer_common/sanitizer_linux.cc b/lib/sanitizer_common/sanitizer_linux.cc
index a94a63c7f16d..a79a2a155db9 100644
--- a/lib/sanitizer_common/sanitizer_linux.cc
+++ b/lib/sanitizer_common/sanitizer_linux.cc
@@ -59,6 +59,14 @@
 #include <ucontext.h>
 #include <unistd.h>
 
+#if SANITIZER_LINUX
+#include <sys/utsname.h>
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#include <sys/personality.h>
+#endif
+
 #if SANITIZER_FREEBSD
 #include <sys/exec.h>
 #include <sys/sysctl.h>
@@ -209,7 +217,6 @@ static void stat64_to_stat(struct stat64 *in, struct stat *out) {
   out->st_atime = in->st_atime;
   out->st_mtime = in->st_mtime;
   out->st_ctime = in->st_ctime;
-  out->st_ino = in->st_ino;
 }
 #endif
 
@@ -229,7 +236,6 @@ static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) {
   out->st_atime = in->st_atime_nsec;
   out->st_mtime = in->st_mtime_nsec;
   out->st_ctime = in->st_ctime_nsec;
-  out->st_ino = in->st_ino;
 }
 #endif
 
@@ -815,6 +821,72 @@ bool ThreadLister::GetDirectoryEntries() {
   return true;
 }
 
+#if SANITIZER_WORDSIZE == 32
+// Take care of unusable kernel area in top gigabyte.
+static uptr GetKernelAreaSize() {
+#if SANITIZER_LINUX && !SANITIZER_X32
+  const uptr gbyte = 1UL << 30;
+
+  // Firstly check if there are writable segments
+  // mapped to top gigabyte (e.g. stack).
+  MemoryMappingLayout proc_maps(/*cache_enabled*/true);
+  MemoryMappedSegment segment;
+  while (proc_maps.Next(&segment)) {
+    if ((segment.end >= 3 * gbyte) && segment.IsWritable()) return 0;
+  }
+
+#if !SANITIZER_ANDROID
+  // Even if nothing is mapped, top Gb may still be accessible
+  // if we are running on 64-bit kernel.
+  // Uname may report misleading results if personality type
+  // is modified (e.g. under schroot) so check this as well.
+  struct utsname uname_info;
+  int pers = personality(0xffffffffUL);
+  if (!(pers & PER_MASK)
+      && uname(&uname_info) == 0
+      && internal_strstr(uname_info.machine, "64"))
+    return 0;
+#endif  // SANITIZER_ANDROID
+
+  // Top gigabyte is reserved for kernel.
+  return gbyte;
+#else
+  return 0;
+#endif  // SANITIZER_LINUX && !SANITIZER_X32
+}
+#endif  // SANITIZER_WORDSIZE == 32
+
+uptr GetMaxVirtualAddress() {
+#if SANITIZER_WORDSIZE == 64
+# if defined(__powerpc64__) || defined(__aarch64__)
+  // On PowerPC64 we have two different address space layouts: 44- and 46-bit.
+  // We somehow need to figure out which one we are using now and choose
+  // one of 0x00000fffffffffffUL and 0x00003fffffffffffUL.
+  // Note that with 'ulimit -s unlimited' the stack is moved away from the top
+  // of the address space, so simply checking the stack address is not enough.
+  // This should (does) work for both PowerPC64 Endian modes.
+  // Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit.
+  return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;
+# elif defined(__mips64)
+  return (1ULL << 40) - 1;  // 0x000000ffffffffffUL;
+# elif defined(__s390x__)
+  return (1ULL << 53) - 1;  // 0x001fffffffffffffUL;
+# else
+  return (1ULL << 47) - 1;  // 0x00007fffffffffffUL;
+# endif
+#else  // SANITIZER_WORDSIZE == 32
+# if defined(__s390__)
+  return (1ULL << 31) - 1;  // 0x7fffffff;
+# else
+  uptr res = (1ULL << 32) - 1;  // 0xffffffff;
+  if (!common_flags()->full_address_space)
+    res -= GetKernelAreaSize();
+  CHECK_LT(reinterpret_cast<uptr>(&res), res);
+  return res;
+# endif
+#endif  // SANITIZER_WORDSIZE
+}
+
 uptr GetPageSize() {
 // Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array.
 #if SANITIZER_ANDROID
@@ -1599,7 +1671,8 @@ void CheckNoDeepBind(const char *filename, int flag) {
 #endif
 }
 
-uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding) {
+uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
+                              uptr *largest_gap_found) {
   UNREACHABLE("FindAvailableMemoryRange is not available");
   return 0;
 }
diff --git a/lib/sanitizer_common/sanitizer_linux_libcdep.cc b/lib/sanitizer_common/sanitizer_linux_libcdep.cc
index b9a48a1e496b..52196db12731 100644
--- a/lib/sanitizer_common/sanitizer_linux_libcdep.cc
+++ b/lib/sanitizer_common/sanitizer_linux_libcdep.cc
@@ -81,28 +81,25 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
 
     // Find the mapping that contains a stack variable.
     MemoryMappingLayout proc_maps(/*cache_enabled*/true);
-    uptr start, end, offset;
+    MemoryMappedSegment segment;
     uptr prev_end = 0;
-    while (proc_maps.Next(&start, &end, &offset, nullptr, 0,
-          /* protection */nullptr)) {
-      if ((uptr)&rl < end)
-        break;
-      prev_end = end;
+    while (proc_maps.Next(&segment)) {
+      if ((uptr)&rl < segment.end) break;
+      prev_end = segment.end;
     }
-    CHECK((uptr)&rl >= start && (uptr)&rl < end);
+    CHECK((uptr)&rl >= segment.start && (uptr)&rl < segment.end);
 
     // Get stacksize from rlimit, but clip it so that it does not overlap
     // with other mappings.
     uptr stacksize = rl.rlim_cur;
-    if (stacksize > end - prev_end)
-      stacksize = end - prev_end;
+    if (stacksize > segment.end - prev_end) stacksize = segment.end - prev_end;
     // When running with unlimited stack size, we still want to set some limit.
     // The unlimited stack size is caused by 'ulimit -s unlimited'.
     // Also, for some reason, GNU make spawns subprocesses with unlimited stack.
     if (stacksize > kMaxThreadStackSize)
       stacksize = kMaxThreadStackSize;
-    *stack_top = end;
-    *stack_bottom = end - stacksize;
+    *stack_top = segment.end;
+    *stack_bottom = segment.end - stacksize;
     return;
   }
   pthread_attr_t attr;
diff --git a/lib/sanitizer_common/sanitizer_mac.cc b/lib/sanitizer_common/sanitizer_mac.cc
index b48238106dd9..8df01815f9f7 100644
--- a/lib/sanitizer_common/sanitizer_mac.cc
+++ b/lib/sanitizer_common/sanitizer_mac.cc
@@ -191,7 +191,8 @@ void internal_sigfillset(__sanitizer_sigset_t *set) { sigfillset(set); }
 
 uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
                           __sanitizer_sigset_t *oldset) {
-  return sigprocmask(how, set, oldset);
+  // Don't use sigprocmask here, because it affects all threads.
+  return pthread_sigmask(how, set, oldset);
 }
 
 // Doesn't call pthread_atfork() handlers (but not available on 10.6).
@@ -799,9 +800,48 @@ char **GetArgv() {
   return *_NSGetArgv();
 }
 
+#if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM
+// The task_vm_info struct is normally provided by the macOS SDK, but we need
+// fields only available in 10.12+. Declare the struct manually to be able to
+// build against older SDKs.
+struct __sanitizer_task_vm_info {
+  uptr _unused[(SANITIZER_WORDSIZE == 32) ? 20 : 19];
+  uptr min_address;
+  uptr max_address;
+};
+
+uptr GetTaskInfoMaxAddress() {
+  __sanitizer_task_vm_info vm_info = {{0}, 0, 0};
+  mach_msg_type_number_t count = sizeof(vm_info) / sizeof(int);
+  int err = task_info(mach_task_self(), TASK_VM_INFO, (int *)&vm_info, &count);
+  if (err == 0) {
+    return vm_info.max_address - 1;
+  } else {
+    // xnu cannot provide vm address limit
+    return 0x200000000 - 1;
+  }
+}
+#endif
+
+uptr GetMaxVirtualAddress() {
+#if SANITIZER_WORDSIZE == 64
+# if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM
+  // Get the maximum VM address
+  static uptr max_vm = GetTaskInfoMaxAddress();
+  CHECK(max_vm);
+  return max_vm;
+# else
+  return (1ULL << 47) - 1;  // 0x00007fffffffffffUL;
+# endif
+#else  // SANITIZER_WORDSIZE == 32
+  return (1ULL << 32) - 1;  // 0xffffffff;
+#endif  // SANITIZER_WORDSIZE
+}
+
 uptr FindAvailableMemoryRange(uptr shadow_size,
                               uptr alignment,
-                              uptr left_padding) {
+                              uptr left_padding,
+                              uptr *largest_gap_found) {
   typedef vm_region_submap_short_info_data_64_t RegionInfo;
   enum { kRegionInfoSize = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64 };
   // Start searching for available memory region past PAGEZERO, which is
@@ -812,6 +852,7 @@ uptr FindAvailableMemoryRange(uptr shadow_size,
   mach_vm_address_t address = start_address;
   mach_vm_address_t free_begin = start_address;
   kern_return_t kr = KERN_SUCCESS;
+  if (largest_gap_found) *largest_gap_found = 0;
   while (kr == KERN_SUCCESS) {
     mach_vm_size_t vmsize = 0;
     natural_t depth = 0;
@@ -821,10 +862,15 @@ uptr FindAvailableMemoryRange(uptr shadow_size,
                                 (vm_region_info_t)&vminfo, &count);
     if (free_begin != address) {
       // We found a free region [free_begin..address-1].
-      uptr shadow_address = RoundUpTo((uptr)free_begin + left_padding,
-                                      alignment);
-      if (shadow_address + shadow_size < (uptr)address) {
-        return shadow_address;
+      uptr gap_start = RoundUpTo((uptr)free_begin + left_padding, alignment);
+      uptr gap_end = RoundDownTo((uptr)address, alignment);
+      uptr gap_size = gap_end > gap_start ? gap_end - gap_start : 0;
+      if (shadow_size < gap_size) {
+        return gap_start;
+      }
+
+      if (largest_gap_found && *largest_gap_found < gap_size) {
+        *largest_gap_found = gap_size;
       }
     }
     // Move to the next region.
diff --git a/lib/sanitizer_common/sanitizer_mac.h b/lib/sanitizer_common/sanitizer_mac.h
index 636d9bfeac8c..3f1c68c8610a 100644
--- a/lib/sanitizer_common/sanitizer_mac.h
+++ b/lib/sanitizer_common/sanitizer_mac.h
@@ -36,6 +36,8 @@ MacosVersion GetMacosVersion();
 
 char **GetEnviron();
 
+void RestrictMemoryToMaxAddress(uptr max_address);
+
 }  // namespace __sanitizer
 
 extern "C" {
diff --git a/lib/sanitizer_common/sanitizer_mac_libcdep.cc b/lib/sanitizer_common/sanitizer_mac_libcdep.cc
new file mode 100644
index 000000000000..c95daa9372ad
--- /dev/null
+++ b/lib/sanitizer_common/sanitizer_mac_libcdep.cc
@@ -0,0 +1,30 @@
+//===-- sanitizer_mac_libcdep.cc ------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries and
+// implements OSX-specific functions.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_MAC
+#include "sanitizer_mac.h"
+
+#include <sys/mman.h>
+
+namespace __sanitizer {
+
+void RestrictMemoryToMaxAddress(uptr max_address) {
+  uptr size_to_mmap = GetMaxVirtualAddress() + 1 - max_address;
+  void *res = MmapFixedNoAccess(max_address, size_to_mmap, "high gap");
+  CHECK(res != MAP_FAILED);
+}
+
+}  // namespace __sanitizer
+
+#endif  // SANITIZER_MAC
diff --git a/lib/sanitizer_common/sanitizer_platform_limits_posix.cc b/lib/sanitizer_common/sanitizer_platform_limits_posix.cc
index 683f019d70c3..83f4fd22f623 100644
--- a/lib/sanitizer_common/sanitizer_platform_limits_posix.cc
+++ b/lib/sanitizer_common/sanitizer_platform_limits_posix.cc
@@ -25,7 +25,6 @@
 #endif
 #include <arpa/inet.h>
 #include <dirent.h>
-#include <errno.h>
 #include <grp.h>
 #include <limits.h>
 #include <net/if.h>
@@ -931,14 +930,6 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
   unsigned IOCTL_SNDCTL_DSP_GETOSPACE = SNDCTL_DSP_GETOSPACE;
 #endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID
 
-  const int errno_EINVAL = EINVAL;
-// EOWNERDEAD is not present in some older platforms.
-#if defined(EOWNERDEAD)
-  const int errno_EOWNERDEAD = EOWNERDEAD;
-#else
-  const int errno_EOWNERDEAD = -1;
-#endif
-
   const int si_SEGV_MAPERR = SEGV_MAPERR;
   const int si_SEGV_ACCERR = SEGV_ACCERR;
 } // namespace __sanitizer
diff --git a/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/lib/sanitizer_common/sanitizer_platform_limits_posix.h
index 24ffcd7d94f4..63dcd2a6d683 100644
--- a/lib/sanitizer_common/sanitizer_platform_limits_posix.h
+++ b/lib/sanitizer_common/sanitizer_platform_limits_posix.h
@@ -1464,9 +1464,6 @@ struct __sanitizer_cookie_io_functions_t {
   extern unsigned IOCTL_PIO_SCRNMAP;
 #endif
 
-  extern const int errno_EINVAL;
-  extern const int errno_EOWNERDEAD;
-
   extern const int si_SEGV_MAPERR;
   extern const int si_SEGV_ACCERR;
 }  // namespace __sanitizer
diff --git a/lib/sanitizer_common/sanitizer_posix.cc b/lib/sanitizer_common/sanitizer_posix.cc
index 63f1bf713b24..8d3128ae199d 100644
--- a/lib/sanitizer_common/sanitizer_posix.cc
+++ b/lib/sanitizer_common/sanitizer_posix.cc
@@ -27,14 +27,6 @@
 #include <signal.h>
 #include <sys/mman.h>
 
-#if SANITIZER_LINUX
-#include <sys/utsname.h>
-#endif
-
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
-#include <sys/personality.h>
-#endif
-
 #if SANITIZER_FREEBSD
 // The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before
 // that, it was never implemented.  So just define it to zero.
@@ -49,80 +41,6 @@ uptr GetMmapGranularity() {
   return GetPageSize();
 }
 
-#if SANITIZER_WORDSIZE == 32
-// Take care of unusable kernel area in top gigabyte.
-static uptr GetKernelAreaSize() {
-#if SANITIZER_LINUX && !SANITIZER_X32
-  const uptr gbyte = 1UL << 30;
-
-  // Firstly check if there are writable segments
-  // mapped to top gigabyte (e.g. stack).
-  MemoryMappingLayout proc_maps(/*cache_enabled*/true);
-  uptr end, prot;
-  while (proc_maps.Next(/*start*/nullptr, &end,
-                        /*offset*/nullptr, /*filename*/nullptr,
-                        /*filename_size*/0, &prot)) {
-    if ((end >= 3 * gbyte)
-        && (prot & MemoryMappingLayout::kProtectionWrite) != 0)
-      return 0;
-  }
-
-#if !SANITIZER_ANDROID
-  // Even if nothing is mapped, top Gb may still be accessible
-  // if we are running on 64-bit kernel.
-  // Uname may report misleading results if personality type
-  // is modified (e.g. under schroot) so check this as well.
-  struct utsname uname_info;
-  int pers = personality(0xffffffffUL);
-  if (!(pers & PER_MASK)
-      && uname(&uname_info) == 0
-      && internal_strstr(uname_info.machine, "64"))
-    return 0;
-#endif  // SANITIZER_ANDROID
-
-  // Top gigabyte is reserved for kernel.
-  return gbyte;
-#else
-  return 0;
-#endif  // SANITIZER_LINUX && !SANITIZER_X32
-}
-#endif  // SANITIZER_WORDSIZE == 32
-
-uptr GetMaxVirtualAddress() {
-#if SANITIZER_WORDSIZE == 64
-# if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM
-  // Ideally, we would derive the upper bound from MACH_VM_MAX_ADDRESS. The
-  // upper bound can change depending on the device.
-  return 0x200000000 - 1;
-# elif defined(__powerpc64__) || defined(__aarch64__)
-  // On PowerPC64 we have two different address space layouts: 44- and 46-bit.
-  // We somehow need to figure out which one we are using now and choose
-  // one of 0x00000fffffffffffUL and 0x00003fffffffffffUL.
-  // Note that with 'ulimit -s unlimited' the stack is moved away from the top
-  // of the address space, so simply checking the stack address is not enough.
-  // This should (does) work for both PowerPC64 Endian modes.
-  // Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit.
-  return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;
-# elif defined(__mips64)
-  return (1ULL << 40) - 1;  // 0x000000ffffffffffUL;
-# elif defined(__s390x__)
-  return (1ULL << 53) - 1;  // 0x001fffffffffffffUL;
-# else
-  return (1ULL << 47) - 1;  // 0x00007fffffffffffUL;
-# endif
-#else  // SANITIZER_WORDSIZE == 32
-# if defined(__s390__)
-  return (1ULL << 31) - 1;  // 0x7fffffff;
-# else
-  uptr res = (1ULL << 32) - 1;  // 0xffffffff;
-  if (!common_flags()->full_address_space)
-    res -= GetKernelAreaSize();
-  CHECK_LT(reinterpret_cast<uptr>(&res), res);
-  return res;
-# endif
-#endif  // SANITIZER_WORDSIZE
-}
-
 void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
   size = RoundUpTo(size, GetPageSizeCached());
   uptr res = internal_mmap(nullptr, size,
@@ -162,7 +80,7 @@ void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
 }
 
 // We want to map a chunk of address space aligned to 'alignment'.
-// We do it by maping a bit more and then unmaping redundant pieces.
+// We do it by mapping a bit more and then unmapping redundant pieces.
 // We probably can do it with fewer syscalls in some OS-dependent way.
 void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
                                    const char *mem_type) {
@@ -313,13 +231,12 @@ static inline bool IntervalsAreSeparate(uptr start1, uptr end1,
 // memory).
 bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
   MemoryMappingLayout proc_maps(/*cache_enabled*/true);
-  uptr start, end;
-  while (proc_maps.Next(&start, &end,
-                        /*offset*/nullptr, /*filename*/nullptr,
-                        /*filename_size*/0, /*protection*/nullptr)) {
-    if (start == end) continue;  // Empty range.
-    CHECK_NE(0, end);
-    if (!IntervalsAreSeparate(start, end - 1, range_start, range_end))
+  MemoryMappedSegment segment;
+  while (proc_maps.Next(&segment)) {
+    if (segment.start == segment.end) continue;  // Empty range.
+    CHECK_NE(0, segment.end);
+    if (!IntervalsAreSeparate(segment.start, segment.end - 1, range_start,
+                              range_end))
       return false;
   }
   return true;
@@ -327,13 +244,13 @@ bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
 
 void DumpProcessMap() {
   MemoryMappingLayout proc_maps(/*cache_enabled*/true);
-  uptr start, end;
   const sptr kBufSize = 4095;
   char *filename = (char*)MmapOrDie(kBufSize, __func__);
+  MemoryMappedSegment segment(filename, kBufSize);
   Report("Process memory map follows:\n");
-  while (proc_maps.Next(&start, &end, /* file_offset */nullptr,
-                        filename, kBufSize, /* protection */nullptr)) {
-    Printf("\t%p-%p\t%s\n", (void*)start, (void*)end, filename);
+  while (proc_maps.Next(&segment)) {
+    Printf("\t%p-%p\t%s\n", (void *)segment.start, (void *)segment.end,
+           segment.filename);
   }
   Report("End of process memory map.\n");
   UnmapOrDie(filename, kBufSize);
@@ -363,14 +280,14 @@ void ReportFile::Write(const char *buffer, uptr length) {
 }
 
 bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end) {
-  uptr s, e, off, prot;
-  InternalScopedString buff(kMaxPathLength);
   MemoryMappingLayout proc_maps(/*cache_enabled*/false);
-  while (proc_maps.Next(&s, &e, &off, buff.data(), buff.size(), &prot)) {
-    if ((prot & MemoryMappingLayout::kProtectionExecute) != 0
-        && internal_strcmp(module, buff.data()) == 0) {
-      *start = s;
-      *end = e;
+  InternalScopedString buff(kMaxPathLength);
+  MemoryMappedSegment segment(buff.data(), kMaxPathLength);
+  while (proc_maps.Next(&segment)) {
+    if (segment.IsExecutable() &&
+        internal_strcmp(module, segment.filename) == 0) {
+      *start = segment.start;
+      *end = segment.end;
       return true;
     }
   }
diff --git a/lib/sanitizer_common/sanitizer_procmaps.h b/lib/sanitizer_common/sanitizer_procmaps.h
index 5aad6b959ad4..06d072b4dc77 100644
--- a/lib/sanitizer_common/sanitizer_procmaps.h
+++ b/lib/sanitizer_common/sanitizer_procmaps.h
@@ -31,13 +31,37 @@ struct ProcSelfMapsBuff {
 void ReadProcMaps(ProcSelfMapsBuff *proc_maps);
 #endif  // SANITIZER_FREEBSD || SANITIZER_LINUX
 
+// Memory protection masks.
+static const uptr kProtectionRead = 1;
+static const uptr kProtectionWrite = 2;
+static const uptr kProtectionExecute = 4;
+static const uptr kProtectionShared = 8;
+
+struct MemoryMappedSegment {
+  MemoryMappedSegment(char *buff = nullptr, uptr size = 0)
+      : filename(buff), filename_size(size) {}
+  ~MemoryMappedSegment() {}
+
+  bool IsReadable() { return protection & kProtectionRead; }
+  bool IsWritable() { return protection & kProtectionWrite; }
+  bool IsExecutable() { return protection & kProtectionExecute; }
+  bool IsShared() { return protection & kProtectionShared; }
+
+  uptr start;
+  uptr end;
+  uptr offset;
+  char *filename;  // owned by caller
+  uptr filename_size;
+  uptr protection;
+  ModuleArch arch;
+  u8 uuid[kModuleUUIDSize];
+};
+
 class MemoryMappingLayout {
  public:
   explicit MemoryMappingLayout(bool cache_enabled);
   ~MemoryMappingLayout();
-  bool Next(uptr *start, uptr *end, uptr *offset, char filename[],
-            uptr filename_size, uptr *protection, ModuleArch *arch = nullptr,
-            u8 *uuid = nullptr);
+  bool Next(MemoryMappedSegment *segment);
   void Reset();
   // In some cases, e.g. when running under a sandbox on Linux, ASan is unable
   // to obtain the memory mappings. It should fall back to pre-cached data
@@ -47,12 +71,6 @@ class MemoryMappingLayout {
   // Adds all mapped objects into a vector.
   void DumpListOfModules(InternalMmapVector<LoadedModule> *modules);
 
-  // Memory protection masks.
-  static const uptr kProtectionRead = 1;
-  static const uptr kProtectionWrite = 2;
-  static const uptr kProtectionExecute = 4;
-  static const uptr kProtectionShared = 8;
-
  private:
   void LoadFromCache();
 
@@ -67,10 +85,7 @@ class MemoryMappingLayout {
   static StaticSpinMutex cache_lock_;  // protects cached_proc_self_maps_.
 # elif SANITIZER_MAC
   template <u32 kLCSegment, typename SegmentCommand>
-  bool NextSegmentLoad(uptr *start, uptr *end, uptr *offset, char filename[],
-                       uptr filename_size, ModuleArch *arch, u8 *uuid,
-                       uptr *protection);
-  void GetSegmentAddrRange(uptr *start, uptr *end, uptr vmaddr, uptr vmsize);
+  bool NextSegmentLoad(MemoryMappedSegment *segment);
   int current_image_;
   u32 current_magic_;
   u32 current_filetype_;
diff --git a/lib/sanitizer_common/sanitizer_procmaps_common.cc b/lib/sanitizer_common/sanitizer_procmaps_common.cc
index c583f42f25d8..b95f301a437d 100644
--- a/lib/sanitizer_common/sanitizer_procmaps_common.cc
+++ b/lib/sanitizer_common/sanitizer_procmaps_common.cc
@@ -119,12 +119,10 @@ void MemoryMappingLayout::LoadFromCache() {
 void MemoryMappingLayout::DumpListOfModules(
     InternalMmapVector<LoadedModule> *modules) {
   Reset();
-  uptr cur_beg, cur_end, cur_offset, prot;
   InternalScopedString module_name(kMaxPathLength);
-  for (uptr i = 0; Next(&cur_beg, &cur_end, &cur_offset, module_name.data(),
-                        module_name.size(), &prot);
-       i++) {
-    const char *cur_name = module_name.data();
+  MemoryMappedSegment segment(module_name.data(), module_name.size());
+  for (uptr i = 0; Next(&segment); i++) {
+    const char *cur_name = segment.filename;
     if (cur_name[0] == '\0')
       continue;
     // Don't subtract 'cur_beg' from the first entry:
@@ -138,11 +136,11 @@ void MemoryMappingLayout::DumpListOfModules(
     //   mapped high at address space (in particular, higher than
     //   shadow memory of the tool), so the module can't be the
     //   first entry.
-    uptr base_address = (i ? cur_beg : 0) - cur_offset;
+    uptr base_address = (i ? segment.start : 0) - segment.offset;
     LoadedModule cur_module;
     cur_module.set(cur_name, base_address);
-    cur_module.addAddressRange(cur_beg, cur_end, prot & kProtectionExecute,
-                               prot & kProtectionWrite);
+    cur_module.addAddressRange(segment.start, segment.end,
+                               segment.IsExecutable(), segment.IsWritable());
     modules->push_back(cur_module);
   }
 }
diff --git a/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc b/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc
index 30216456330e..f0cdbeb4483a 100644
--- a/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc
+++ b/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc
@@ -48,36 +48,27 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
   proc_maps->len = Size;
 }
 
-bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
-                               char filename[], uptr filename_size,
-                               uptr *protection, ModuleArch *arch, u8 *uuid) {
-  CHECK(!arch && "not implemented");
-  CHECK(!uuid && "not implemented");
+bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
   char *last = proc_self_maps_.data + proc_self_maps_.len;
   if (current_ >= last) return false;
-  uptr dummy;
-  if (!start) start = &dummy;
-  if (!end) end = &dummy;
-  if (!offset) offset = &dummy;
-  if (!protection) protection = &dummy;
   struct kinfo_vmentry *VmEntry = (struct kinfo_vmentry*)current_;
 
-  *start = (uptr)VmEntry->kve_start;
-  *end = (uptr)VmEntry->kve_end;
-  *offset = (uptr)VmEntry->kve_offset;
+  segment->start = (uptr)VmEntry->kve_start;
+  segment->end = (uptr)VmEntry->kve_end;
+  segment->offset = (uptr)VmEntry->kve_offset;
 
-  *protection = 0;
+  segment->protection = 0;
   if ((VmEntry->kve_protection & KVME_PROT_READ) != 0)
-    *protection |= kProtectionRead;
+    segment->protection |= kProtectionRead;
   if ((VmEntry->kve_protection & KVME_PROT_WRITE) != 0)
-    *protection |= kProtectionWrite;
+    segment->protection |= kProtectionWrite;
   if ((VmEntry->kve_protection & KVME_PROT_EXEC) != 0)
-    *protection |= kProtectionExecute;
+    segment->protection |= kProtectionExecute;
 
-  if (filename != NULL && filename_size > 0) {
-    internal_snprintf(filename,
-                      Min(filename_size, (uptr)PATH_MAX),
-                      "%s", VmEntry->kve_path);
+  if (segment->filename != NULL && segment->filename_size > 0) {
+    internal_snprintf(segment->filename,
+                      Min(segment->filename_size, (uptr)PATH_MAX), "%s",
+                      VmEntry->kve_path);
   }
 
   current_ += VmEntry->kve_structsize;
diff --git a/lib/sanitizer_common/sanitizer_procmaps_linux.cc b/lib/sanitizer_common/sanitizer_procmaps_linux.cc
index 7e4a44be95b6..1bcad2bf70e6 100644
--- a/lib/sanitizer_common/sanitizer_procmaps_linux.cc
+++ b/lib/sanitizer_common/sanitizer_procmaps_linux.cc
@@ -26,41 +26,28 @@ static bool IsOneOf(char c, char c1, char c2) {
   return c == c1 || c == c2;
 }
 
-bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
-                               char filename[], uptr filename_size,
-                               uptr *protection, ModuleArch *arch, u8 *uuid) {
-  CHECK(!arch && "not implemented");
-  CHECK(!uuid && "not implemented");
+bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
   char *last = proc_self_maps_.data + proc_self_maps_.len;
   if (current_ >= last) return false;
-  uptr dummy;
-  if (!start) start = &dummy;
-  if (!end) end = &dummy;
-  if (!offset) offset = &dummy;
-  if (!protection) protection = &dummy;
   char *next_line = (char*)internal_memchr(current_, '\n', last - current_);
   if (next_line == 0)
     next_line = last;
   // Example: 08048000-08056000 r-xp 00000000 03:0c 64593   /foo/bar
-  *start = ParseHex(&current_);
+  segment->start = ParseHex(&current_);
   CHECK_EQ(*current_++, '-');
-  *end = ParseHex(&current_);
+  segment->end = ParseHex(&current_);
   CHECK_EQ(*current_++, ' ');
   CHECK(IsOneOf(*current_, '-', 'r'));
-  *protection = 0;
-  if (*current_++ == 'r')
-    *protection |= kProtectionRead;
+  segment->protection = 0;
+  if (*current_++ == 'r') segment->protection |= kProtectionRead;
   CHECK(IsOneOf(*current_, '-', 'w'));
-  if (*current_++ == 'w')
-    *protection |= kProtectionWrite;
+  if (*current_++ == 'w') segment->protection |= kProtectionWrite;
   CHECK(IsOneOf(*current_, '-', 'x'));
-  if (*current_++ == 'x')
-    *protection |= kProtectionExecute;
+  if (*current_++ == 'x') segment->protection |= kProtectionExecute;
   CHECK(IsOneOf(*current_, 's', 'p'));
-  if (*current_++ == 's')
-    *protection |= kProtectionShared;
+  if (*current_++ == 's') segment->protection |= kProtectionShared;
   CHECK_EQ(*current_++, ' ');
-  *offset = ParseHex(&current_);
+  segment->offset = ParseHex(&current_);
   CHECK_EQ(*current_++, ' ');
   ParseHex(&current_);
   CHECK_EQ(*current_++, ':');
@@ -75,14 +62,12 @@ bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
   while (current_ < next_line && *current_ == ' ')
     current_++;
   // Fill in the filename.
-  uptr i = 0;
-  while (current_ < next_line) {
-    if (filename && i < filename_size - 1)
-      filename[i++] = *current_;
-    current_++;
+  if (segment->filename) {
+    uptr len = Min((uptr)(next_line - current_), segment->filename_size - 1);
+    internal_strncpy(segment->filename, current_, len);
+    segment->filename[len] = 0;
   }
-  if (filename && i < filename_size)
-    filename[i] = 0;
+
   current_ = next_line + 1;
   return true;
 }
diff --git a/lib/sanitizer_common/sanitizer_procmaps_mac.cc b/lib/sanitizer_common/sanitizer_procmaps_mac.cc
index 131017458d4c..560451a16d90 100644
--- a/lib/sanitizer_common/sanitizer_procmaps_mac.cc
+++ b/lib/sanitizer_common/sanitizer_procmaps_mac.cc
@@ -88,6 +88,48 @@ void MemoryMappingLayout::LoadFromCache() {
   // No-op on Mac for now.
 }
 
+// _dyld_get_image_header() and related APIs don't report dyld itself.
+// We work around this by manually recursing through the memory map
+// until we hit a Mach header matching dyld instead. These recurse
+// calls are expensive, but the first memory map generation occurs
+// early in the process, when dyld is one of the only images loaded,
+// so it will be hit after only a few iterations.
+static mach_header *get_dyld_image_header() {
+  mach_port_name_t port;
+  if (task_for_pid(mach_task_self(), internal_getpid(), &port) !=
+      KERN_SUCCESS) {
+    return nullptr;
+  }
+
+  unsigned depth = 1;
+  vm_size_t size = 0;
+  vm_address_t address = 0;
+  kern_return_t err = KERN_SUCCESS;
+  mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
+
+  while (true) {
+    struct vm_region_submap_info_64 info;
+    err = vm_region_recurse_64(port, &address, &size, &depth,
+                               (vm_region_info_t)&info, &count);
+    if (err != KERN_SUCCESS) return nullptr;
+
+    if (size >= sizeof(mach_header) && info.protection & kProtectionRead) {
+      mach_header *hdr = (mach_header *)address;
+      if ((hdr->magic == MH_MAGIC || hdr->magic == MH_MAGIC_64) &&
+          hdr->filetype == MH_DYLINKER) {
+        return hdr;
+      }
+    }
+    address += size;
+  }
+}
+
+const mach_header *get_dyld_hdr() {
+  if (!dyld_hdr) dyld_hdr = get_dyld_image_header();
+
+  return dyld_hdr;
+}
+
 // Next and NextSegmentLoad were inspired by base/sysinfo.cc in
 // Google Perftools, https://github.com/gperftools/gperftools.
 
@@ -96,40 +138,39 @@ void MemoryMappingLayout::LoadFromCache() {
 // segment.
 // Note that the segment addresses are not necessarily sorted.
 template <u32 kLCSegment, typename SegmentCommand>
-bool MemoryMappingLayout::NextSegmentLoad(uptr *start, uptr *end, uptr *offset,
-                                          char filename[], uptr filename_size,
-                                          ModuleArch *arch, u8 *uuid,
-                                          uptr *protection) {
+bool MemoryMappingLayout::NextSegmentLoad(MemoryMappedSegment *segment) {
   const char *lc = current_load_cmd_addr_;
   current_load_cmd_addr_ += ((const load_command *)lc)->cmdsize;
   if (((const load_command *)lc)->cmd == kLCSegment) {
     const SegmentCommand* sc = (const SegmentCommand *)lc;
-    GetSegmentAddrRange(start, end, sc->vmaddr, sc->vmsize);
-    if (protection) {
-      // Return the initial protection.
-      *protection = sc->initprot;
-    }
-    if (offset) {
-      if (current_filetype_ == /*MH_EXECUTE*/ 0x2) {
-        *offset = sc->vmaddr;
-      } else {
-        *offset = sc->fileoff;
-      }
-    }
-    if (filename) {
-      if (current_image_ == kDyldImageIdx) {
-        internal_strncpy(filename, kDyldPath, filename_size);
-      } else {
-        internal_strncpy(filename, _dyld_get_image_name(current_image_),
-                         filename_size);
-      }
-    }
-    if (arch) {
-      *arch = current_arch_;
+
+    if (current_image_ == kDyldImageIdx) {
+      // vmaddr is masked with 0xfffff because on macOS versions < 10.12,
+      // it contains an absolute address rather than an offset for dyld.
+      // To make matters even more complicated, this absolute address
+      // isn't actually the absolute segment address, but the offset portion
+      // of the address is accurate when combined with the dyld base address,
+      // and the mask will give just this offset.
+      segment->start = (sc->vmaddr & 0xfffff) + (uptr)get_dyld_hdr();
+      segment->end = (sc->vmaddr & 0xfffff) + sc->vmsize + (uptr)get_dyld_hdr();
+    } else {
+      const sptr dlloff = _dyld_get_image_vmaddr_slide(current_image_);
+      segment->start = sc->vmaddr + dlloff;
+      segment->end = sc->vmaddr + sc->vmsize + dlloff;
     }
-    if (uuid) {
-      internal_memcpy(uuid, current_uuid_, kModuleUUIDSize);
+
+    // Return the initial protection.
+    segment->protection = sc->initprot;
+    segment->offset =
+        (current_filetype_ == /*MH_EXECUTE*/ 0x2) ? sc->vmaddr : sc->fileoff;
+    if (segment->filename) {
+      const char *src = (current_image_ == kDyldImageIdx)
+                            ? kDyldPath
+                            : _dyld_get_image_name(current_image_);
+      internal_strncpy(segment->filename, src, segment->filename_size);
     }
+    segment->arch = current_arch_;
+    internal_memcpy(segment->uuid, current_uuid_, kModuleUUIDSize);
     return true;
   }
   return false;
@@ -190,70 +231,7 @@ static bool IsModuleInstrumented(const load_command *first_lc) {
   return false;
 }
 
-// _dyld_get_image_header() and related APIs don't report dyld itself.
-// We work around this by manually recursing through the memory map
-// until we hit a Mach header matching dyld instead. These recurse
-// calls are expensive, but the first memory map generation occurs
-// early in the process, when dyld is one of the only images loaded,
-// so it will be hit after only a few iterations.
-static mach_header *get_dyld_image_header() {
-  mach_port_name_t port;
-  if (task_for_pid(mach_task_self(), internal_getpid(), &port) !=
-      KERN_SUCCESS) {
-    return nullptr;
-  }
-
-  unsigned depth = 1;
-  vm_size_t size = 0;
-  vm_address_t address = 0;
-  kern_return_t err = KERN_SUCCESS;
-  mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
-
-  while (true) {
-    struct vm_region_submap_info_64 info;
-    err = vm_region_recurse_64(port, &address, &size, &depth,
-                               (vm_region_info_t)&info, &count);
-    if (err != KERN_SUCCESS) return nullptr;
-
-    if (size >= sizeof(mach_header) &&
-        info.protection & MemoryMappingLayout::kProtectionRead) {
-      mach_header *hdr = (mach_header *)address;
-      if ((hdr->magic == MH_MAGIC || hdr->magic == MH_MAGIC_64) &&
-          hdr->filetype == MH_DYLINKER) {
-        return hdr;
-      }
-    }
-    address += size;
-  }
-}
-
-const mach_header *get_dyld_hdr() {
-  if (!dyld_hdr) dyld_hdr = get_dyld_image_header();
-
-  return dyld_hdr;
-}
-
-void MemoryMappingLayout::GetSegmentAddrRange(uptr *start, uptr *end,
-                                              uptr vmaddr, uptr vmsize) {
-  if (current_image_ == kDyldImageIdx) {
-    // vmaddr is masked with 0xfffff because on macOS versions < 10.12,
-    // it contains an absolute address rather than an offset for dyld.
-    // To make matters even more complicated, this absolute address
-    // isn't actually the absolute segment address, but the offset portion
-    // of the address is accurate when combined with the dyld base address,
-    // and the mask will give just this offset.
-    if (start) *start = (vmaddr & 0xfffff) + (uptr)get_dyld_hdr();
-    if (end) *end = (vmaddr & 0xfffff) + vmsize + (uptr)get_dyld_hdr();
-  } else {
-    const sptr dlloff = _dyld_get_image_vmaddr_slide(current_image_);
-    if (start) *start = vmaddr + dlloff;
-    if (end) *end = vmaddr + vmsize + dlloff;
-  }
-}
-
-bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
-                               char filename[], uptr filename_size,
-                               uptr *protection, ModuleArch *arch, u8 *uuid) {
+bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
   for (; current_image_ >= kDyldImageIdx; current_image_--) {
     const mach_header *hdr = (current_image_ == kDyldImageIdx)
                                  ? get_dyld_hdr()
@@ -291,16 +269,13 @@ bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
 #ifdef MH_MAGIC_64
         case MH_MAGIC_64: {
           if (NextSegmentLoad<LC_SEGMENT_64, struct segment_command_64>(
-                  start, end, offset, filename, filename_size, arch, uuid,
-                  protection))
+                  segment))
             return true;
           break;
         }
 #endif
         case MH_MAGIC: {
-          if (NextSegmentLoad<LC_SEGMENT, struct segment_command>(
-                  start, end, offset, filename, filename_size, arch, uuid,
-                  protection))
+          if (NextSegmentLoad<LC_SEGMENT, struct segment_command>(segment))
             return true;
           break;
         }
@@ -315,28 +290,22 @@ bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
 void MemoryMappingLayout::DumpListOfModules(
     InternalMmapVector<LoadedModule> *modules) {
   Reset();
-  uptr cur_beg, cur_end, prot;
-  ModuleArch cur_arch;
-  u8 cur_uuid[kModuleUUIDSize];
   InternalScopedString module_name(kMaxPathLength);
-  for (uptr i = 0; Next(&cur_beg, &cur_end, 0, module_name.data(),
-                        module_name.size(), &prot, &cur_arch, &cur_uuid[0]);
-       i++) {
-    const char *cur_name = module_name.data();
-    if (cur_name[0] == '\0')
-      continue;
+  MemoryMappedSegment segment(module_name.data(), kMaxPathLength);
+  for (uptr i = 0; Next(&segment); i++) {
+    if (segment.filename[0] == '\0') continue;
     LoadedModule *cur_module = nullptr;
     if (!modules->empty() &&
-        0 == internal_strcmp(cur_name, modules->back().full_name())) {
+        0 == internal_strcmp(segment.filename, modules->back().full_name())) {
       cur_module = &modules->back();
     } else {
       modules->push_back(LoadedModule());
       cur_module = &modules->back();
-      cur_module->set(cur_name, cur_beg, cur_arch, cur_uuid,
-                      current_instrumented_);
+      cur_module->set(segment.filename, segment.start, segment.arch,
+                      segment.uuid, current_instrumented_);
     }
-    cur_module->addAddressRange(cur_beg, cur_end, prot & kProtectionExecute,
-                                prot & kProtectionWrite);
+    cur_module->addAddressRange(segment.start, segment.end,
+                                segment.IsExecutable(), segment.IsWritable());
   }
 }
 
diff --git a/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc b/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc
index 36c98d057bd3..747a4a701728 100644
--- a/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc
+++ b/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc
@@ -43,7 +43,8 @@ void StackTrace::Print() const {
       if (dedup_frames-- > 0) {
         if (dedup_token.length())
           dedup_token.append("--");
-        dedup_token.append(cur->info.function);
+        if (cur->info.function != nullptr)
+          dedup_token.append(cur->info.function);
       }
     }
     frames->ClearAll();
diff --git a/lib/sanitizer_common/sanitizer_win.cc b/lib/sanitizer_common/sanitizer_win.cc
index 89d9cf61c3e4..de01e8d119a1 100644
--- a/lib/sanitizer_common/sanitizer_win.cc
+++ b/lib/sanitizer_common/sanitizer_win.cc
@@ -291,7 +291,8 @@ void DontDumpShadowMemory(uptr addr, uptr length) {
   // FIXME: add madvise-analog when we move to 64-bits.
 }
 
-uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding) {
+uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
+                              uptr *largest_gap_found) {
   uptr address = 0;
   while (true) {
     MEMORY_BASIC_INFORMATION info;
diff --git a/lib/scudo/scudo_allocator.cpp b/lib/scudo/scudo_allocator.cpp
index 00fa192181ad..ec9132f90a4f 100644
--- a/lib/scudo/scudo_allocator.cpp
+++ b/lib/scudo/scudo_allocator.cpp
@@ -264,7 +264,7 @@ ScudoQuarantineCache *getQuarantineCache(ScudoThreadContext *ThreadContext) {
       ScudoQuarantineCache *>(ThreadContext->QuarantineCachePlaceHolder);
 }
 
-Xorshift128Plus *getPrng(ScudoThreadContext *ThreadContext) {
+ScudoPrng *getPrng(ScudoThreadContext *ThreadContext) {
   return &ThreadContext->Prng;
 }
 
@@ -283,7 +283,7 @@ struct ScudoAllocator {
   StaticSpinMutex FallbackMutex;
   AllocatorCache FallbackAllocatorCache;
   ScudoQuarantineCache FallbackQuarantineCache;
-  Xorshift128Plus FallbackPrng;
+  ScudoPrng FallbackPrng;
 
   bool DeallocationTypeMismatch;
   bool ZeroContents;
@@ -333,8 +333,8 @@ struct ScudoAllocator {
         static_cast<uptr>(Options.QuarantineSizeMb) << 20,
         static_cast<uptr>(Options.ThreadLocalQuarantineSizeKb) << 10);
     BackendAllocator.InitCache(&FallbackAllocatorCache);
-    FallbackPrng.initFromURandom();
-    Cookie = FallbackPrng.getNext();
+    FallbackPrng.init();
+    Cookie = FallbackPrng.getU64();
   }
 
   // Helper function that checks for a valid Scudo chunk. nullptr isn't.
@@ -373,19 +373,19 @@ struct ScudoAllocator {
     bool FromPrimary = PrimaryAllocator::CanAllocate(AlignedSize, MinAlignment);
 
     void *Ptr;
-    uptr Salt;
+    u8 Salt;
     uptr AllocationSize = FromPrimary ? AlignedSize : NeededSize;
     uptr AllocationAlignment = FromPrimary ? MinAlignment : Alignment;
     ScudoThreadContext *ThreadContext = getThreadContextAndLock();
     if (LIKELY(ThreadContext)) {
-      Salt = getPrng(ThreadContext)->getNext();
+      Salt = getPrng(ThreadContext)->getU8();
       Ptr = BackendAllocator.Allocate(getAllocatorCache(ThreadContext),
                                       AllocationSize, AllocationAlignment,
                                       FromPrimary);
       ThreadContext->unlock();
     } else {
       SpinMutexLock l(&FallbackMutex);
-      Salt = FallbackPrng.getNext();
+      Salt = FallbackPrng.getU8();
       Ptr = BackendAllocator.Allocate(&FallbackAllocatorCache, AllocationSize,
                                       AllocationAlignment, FromPrimary);
     }
@@ -612,7 +612,7 @@ static void initScudoInternal(const AllocatorOptions &Options) {
 
 void ScudoThreadContext::init() {
   getBackendAllocator().InitCache(&Cache);
-  Prng.initFromURandom();
+  Prng.init();
   memset(QuarantineCachePlaceHolder, 0, sizeof(QuarantineCachePlaceHolder));
 }
 
diff --git a/lib/scudo/scudo_tls.h b/lib/scudo/scudo_tls.h
index f6039bebec44..20c49204cf13 100644
--- a/lib/scudo/scudo_tls.h
+++ b/lib/scudo/scudo_tls.h
@@ -30,7 +30,7 @@ namespace __scudo {
 
 struct ALIGNED(64) ScudoThreadContext : public ScudoThreadContextPlatform {
   AllocatorCache Cache;
-  Xorshift128Plus Prng;
+  ScudoPrng Prng;
   uptr QuarantineCachePlaceHolder[4];
   void init();
   void commitBack();
diff --git a/lib/scudo/scudo_utils.cpp b/lib/scudo/scudo_utils.cpp
index 31c391946c15..f7903ff34c73 100644
--- a/lib/scudo/scudo_utils.cpp
+++ b/lib/scudo/scudo_utils.cpp
@@ -123,40 +123,4 @@ bool testCPUFeature(CPUFeature Feature) {
 }
 #endif  // defined(__x86_64__) || defined(__i386__)
 
-// readRetry will attempt to read Count bytes from the Fd specified, and if
-// interrupted will retry to read additional bytes to reach Count.
-static ssize_t readRetry(int Fd, u8 *Buffer, size_t Count) {
-  ssize_t AmountRead = 0;
-  while (static_cast<size_t>(AmountRead) < Count) {
-    ssize_t Result = read(Fd, Buffer + AmountRead, Count - AmountRead);
-    if (Result > 0)
-      AmountRead += Result;
-    else if (!Result)
-      break;
-    else if (errno != EINTR) {
-      AmountRead = -1;
-      break;
-    }
-  }
-  return AmountRead;
-}
-
-static void fillRandom(u8 *Data, ssize_t Size) {
-  int Fd = open("/dev/urandom", O_RDONLY);
-  if (Fd < 0) {
-    dieWithMessage("ERROR: failed to open /dev/urandom.\n");
-  }
-  bool Success = readRetry(Fd, Data, Size) == Size;
-  close(Fd);
-  if (!Success) {
-    dieWithMessage("ERROR: failed to read enough data from /dev/urandom.\n");
-  }
-}
-
-// Seeds the xorshift state with /dev/urandom.
-// TODO(kostyak): investigate using getrandom() if available.
-void Xorshift128Plus::initFromURandom() {
-  fillRandom(reinterpret_cast<u8 *>(State), sizeof(State));
-}
-
 }  // namespace __scudo
diff --git a/lib/scudo/scudo_utils.h b/lib/scudo/scudo_utils.h
index 7198476f42cf..6c6c9d893404 100644
--- a/lib/scudo/scudo_utils.h
+++ b/lib/scudo/scudo_utils.h
@@ -36,23 +36,58 @@ enum CPUFeature {
 };
 bool testCPUFeature(CPUFeature feature);
 
-// Tiny PRNG based on https://en.wikipedia.org/wiki/Xorshift#xorshift.2B
-// The state (128 bits) will be stored in thread local storage.
-struct Xorshift128Plus {
+INLINE u64 rotl(const u64 X, int K) {
+  return (X << K) | (X >> (64 - K));
+}
+
+// XoRoShiRo128+ PRNG (http://xoroshiro.di.unimi.it/).
+struct XoRoShiRo128Plus {
  public:
-  void initFromURandom();
-  u64 getNext() {
-    u64 x = State[0];
-    const u64 y = State[1];
-    State[0] = y;
-    x ^= x << 23;
-    State[1] = x ^ y ^ (x >> 17) ^ (y >> 26);
-    return State[1] + y;
+  void init() {
+    if (UNLIKELY(!GetRandom(reinterpret_cast<void *>(State), sizeof(State)))) {
+      // Early processes (eg: init) do not have /dev/urandom yet, but we still
+      // have to provide them with some degree of entropy. Not having a secure
+      // seed is not as problematic for them, as they are less likely to be
+      // the target of heap based vulnerabilities exploitation attempts.
+      State[0] = NanoTime();
+      State[1] = 0;
+    }
+    fillCache();
   }
+  u8 getU8() {
+    if (UNLIKELY(isCacheEmpty()))
+      fillCache();
+    const u8 Result = static_cast<u8>(CachedBytes & 0xff);
+    CachedBytes >>= 8;
+    CachedBytesAvailable--;
+    return Result;
+  }
+  u64 getU64() { return next(); }
+
  private:
+  u8 CachedBytesAvailable;
+  u64 CachedBytes;
   u64 State[2];
+  u64 next() {
+    const u64 S0 = State[0];
+    u64 S1 = State[1];
+    const u64 Result = S0 + S1;
+    S1 ^= S0;
+    State[0] = rotl(S0, 55) ^ S1 ^ (S1 << 14);
+    State[1] = rotl(S1, 36);
+    return Result;
+  }
+  bool isCacheEmpty() {
+    return CachedBytesAvailable == 0;
+  }
+  void fillCache() {
+    CachedBytes = next();
+    CachedBytesAvailable = sizeof(CachedBytes);
+  }
 };
 
+typedef XoRoShiRo128Plus ScudoPrng;
+
 }  // namespace __scudo
 
 #endif  // SCUDO_UTILS_H_
diff --git a/lib/tsan/CMakeLists.txt b/lib/tsan/CMakeLists.txt
index 195ecb5dfe8a..193158c54e62 100644
--- a/lib/tsan/CMakeLists.txt
+++ b/lib/tsan/CMakeLists.txt
@@ -100,7 +100,7 @@ set(TSAN_RUNTIME_LIBRARIES)
 add_compiler_rt_component(tsan)
 
 if(APPLE)
-  set(TSAN_ASM_SOURCES rtl/tsan_rtl_amd64.S)
+  set(TSAN_ASM_SOURCES rtl/tsan_rtl_amd64.S rtl/tsan_rtl_aarch64.S)
   # Xcode will try to compile this file as C ('clang -x c'), and that will fail.
   if (${CMAKE_GENERATOR} STREQUAL "Xcode")
     enable_language(ASM)
diff --git a/lib/tsan/check_analyze.sh b/lib/tsan/check_analyze.sh
index 22eb444198a4..54dd1b0232dc 100755
--- a/lib/tsan/check_analyze.sh
+++ b/lib/tsan/check_analyze.sh
@@ -34,8 +34,8 @@ done
 
 for f in read1 read2 read4 read8; do
   check $f rsp 1
-  check $f push 4
-  check $f pop 4
+  check $f push 3
+  check $f pop 3
 done
 
 for f in func_entry func_exit; do
diff --git a/lib/tsan/dd/dd_interceptors.cc b/lib/tsan/dd/dd_interceptors.cc
index 97c72dd2b7fd..a39218f0454b 100644
--- a/lib/tsan/dd/dd_interceptors.cc
+++ b/lib/tsan/dd/dd_interceptors.cc
@@ -270,20 +270,19 @@ namespace __dsan {
 
 static void InitDataSeg() {
   MemoryMappingLayout proc_maps(true);
-  uptr start, end, offset;
   char name[128];
+  MemoryMappedSegment segment(name, ARRAY_SIZE(name));
   bool prev_is_data = false;
-  while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name),
-                        /*protection*/ 0)) {
-    bool is_data = offset != 0 && name[0] != 0;
+  while (proc_maps.Next(&segment)) {
+    bool is_data = segment.offset != 0 && segment.filename[0] != 0;
     // BSS may get merged with [heap] in /proc/self/maps. This is not very
     // reliable.
-    bool is_bss = offset == 0 &&
-      (name[0] == 0 || internal_strcmp(name, "[heap]") == 0) && prev_is_data;
-    if (g_data_start == 0 && is_data)
-      g_data_start = start;
-    if (is_bss)
-      g_data_end = end;
+    bool is_bss = segment.offset == 0 &&
+                  (segment.filename[0] == 0 ||
+                   internal_strcmp(segment.filename, "[heap]") == 0) &&
+                  prev_is_data;
+    if (g_data_start == 0 && is_data) g_data_start = segment.start;
+    if (is_bss) g_data_end = segment.end;
     prev_is_data = is_data;
   }
   VPrintf(1, "guessed data_start=%p data_end=%p\n",  g_data_start, g_data_end);
diff --git a/lib/tsan/go/buildgo.sh b/lib/tsan/go/buildgo.sh
index 59176809eae0..617dd9e11d27 100755
--- a/lib/tsan/go/buildgo.sh
+++ b/lib/tsan/go/buildgo.sh
@@ -69,7 +69,7 @@ elif [ "`uname -a | grep FreeBSD`" != "" ]; then
 	"
 elif [ "`uname -a | grep Darwin`" != "" ]; then
 	SUFFIX="darwin_amd64"
-	OSCFLAGS="-fPIC -Wno-unused-const-variable -Wno-unknown-warning-option -mmacosx-version-min=10.7"
+	OSCFLAGS="-fPIC -Wno-unused-const-variable -Wno-unknown-warning-option -isysroot $(xcodebuild -version -sdk macosx Path) -mmacosx-version-min=10.7"
 	OSLDFLAGS="-lpthread -fPIC -fpie -mmacosx-version-min=10.7"
 	SRCS="
 		$SRCS
diff --git a/lib/tsan/rtl/tsan_clock.cc b/lib/tsan/rtl/tsan_clock.cc
index 32435adfdf33..9ee9104283f8 100644
--- a/lib/tsan/rtl/tsan_clock.cc
+++ b/lib/tsan/rtl/tsan_clock.cc
@@ -101,6 +101,9 @@ ThreadClock::ThreadClock(unsigned tid, unsigned reused)
   clk_[tid_].reused = reused_;
 }
 
+void ThreadClock::ResetCached(ClockCache *c) {
+}
+
 void ThreadClock::acquire(ClockCache *c, const SyncClock *src) {
   DCHECK_LE(nclk_, kMaxTid);
   DCHECK_LE(src->size_, kMaxTid);
@@ -116,9 +119,7 @@ void ThreadClock::acquire(ClockCache *c, const SyncClock *src) {
   // Check if we've already acquired src after the last release operation on src
   bool acquired = false;
   if (nclk > tid_) {
-    CPP_STAT_INC(StatClockAcquireLarge);
     if (src->elem(tid_).reused == reused_) {
-      CPP_STAT_INC(StatClockAcquireRepeat);
       for (unsigned i = 0; i < kDirtyTids; i++) {
         unsigned tid = src->dirty_tids_[i];
         if (tid != kInvalidTid) {
@@ -266,11 +267,11 @@ void ThreadClock::UpdateCurrentThread(SyncClock *dst) const {
 
   for (unsigned i = 0; i < kDirtyTids; i++) {
     if (dst->dirty_tids_[i] == tid_) {
-      CPP_STAT_INC(StatClockReleaseFast1);
+      CPP_STAT_INC(StatClockReleaseFast);
       return;
     }
     if (dst->dirty_tids_[i] == kInvalidTid) {
-      CPP_STAT_INC(StatClockReleaseFast2);
+      CPP_STAT_INC(StatClockReleaseFast);
       dst->dirty_tids_[i] = tid_;
       return;
     }
@@ -297,56 +298,9 @@ bool ThreadClock::IsAlreadyAcquired(const SyncClock *src) const {
   return true;
 }
 
-void SyncClock::Resize(ClockCache *c, uptr nclk) {
-  CPP_STAT_INC(StatClockReleaseResize);
-  if (RoundUpTo(nclk, ClockBlock::kClockCount) <=
-      RoundUpTo(size_, ClockBlock::kClockCount)) {
-    // Growing within the same block.
-    // Memory is already allocated, just increase the size.
-    size_ = nclk;
-    return;
-  }
-  if (nclk <= ClockBlock::kClockCount) {
-    // Grow from 0 to one-level table.
-    CHECK_EQ(size_, 0);
-    CHECK_EQ(tab_, 0);
-    CHECK_EQ(tab_idx_, 0);
-    size_ = nclk;
-    tab_idx_ = ctx->clock_alloc.Alloc(c);
-    tab_ = ctx->clock_alloc.Map(tab_idx_);
-    internal_memset(tab_, 0, sizeof(*tab_));
-    return;
-  }
-  // Growing two-level table.
-  if (size_ == 0) {
-    // Allocate first level table.
-    tab_idx_ = ctx->clock_alloc.Alloc(c);
-    tab_ = ctx->clock_alloc.Map(tab_idx_);
-    internal_memset(tab_, 0, sizeof(*tab_));
-  } else if (size_ <= ClockBlock::kClockCount) {
-    // Transform one-level table to two-level table.
-    u32 old = tab_idx_;
-    tab_idx_ = ctx->clock_alloc.Alloc(c);
-    tab_ = ctx->clock_alloc.Map(tab_idx_);
-    internal_memset(tab_, 0, sizeof(*tab_));
-    tab_->table[0] = old;
-  }
-  // At this point we have first level table allocated.
-  // Add second level tables as necessary.
-  for (uptr i = RoundUpTo(size_, ClockBlock::kClockCount);
-      i < nclk; i += ClockBlock::kClockCount) {
-    u32 idx = ctx->clock_alloc.Alloc(c);
-    ClockBlock *cb = ctx->clock_alloc.Map(idx);
-    internal_memset(cb, 0, sizeof(*cb));
-    CHECK_EQ(tab_->table[i/ClockBlock::kClockCount], 0);
-    tab_->table[i/ClockBlock::kClockCount] = idx;
-  }
-  size_ = nclk;
-}
-
 // Sets a single element in the vector clock.
 // This function is called only from weird places like AcquireGlobal.
-void ThreadClock::set(unsigned tid, u64 v) {
+void ThreadClock::set(ClockCache *c, unsigned tid, u64 v) {
   DCHECK_LT(tid, kMaxTid);
   DCHECK_GE(v, clk_[tid].epoch);
   clk_[tid].epoch = v;
@@ -366,14 +320,8 @@ void ThreadClock::DebugDump(int(*printf)(const char *s, ...)) {
       tid_, reused_, last_acquire_);
 }
 
-SyncClock::SyncClock()
-    : release_store_tid_(kInvalidTid)
-    , release_store_reused_()
-    , tab_()
-    , tab_idx_()
-    , size_() {
-  for (uptr i = 0; i < kDirtyTids; i++)
-    dirty_tids_[i] = kInvalidTid;
+SyncClock::SyncClock() {
+  ResetImpl();
 }
 
 SyncClock::~SyncClock() {
@@ -395,6 +343,10 @@ void SyncClock::Reset(ClockCache *c) {
       ctx->clock_alloc.Free(c, tab_->table[i / ClockBlock::kClockCount]);
     ctx->clock_alloc.Free(c, tab_idx_);
   }
+  ResetImpl();
+}
+
+void SyncClock::ResetImpl() {
   tab_ = 0;
   tab_idx_ = 0;
   size_ = 0;
@@ -404,6 +356,53 @@ void SyncClock::Reset(ClockCache *c) {
     dirty_tids_[i] = kInvalidTid;
 }
 
+void SyncClock::Resize(ClockCache *c, uptr nclk) {
+  CPP_STAT_INC(StatClockReleaseResize);
+  if (RoundUpTo(nclk, ClockBlock::kClockCount) <=
+      RoundUpTo(size_, ClockBlock::kClockCount)) {
+    // Growing within the same block.
+    // Memory is already allocated, just increase the size.
+    size_ = nclk;
+    return;
+  }
+  if (nclk <= ClockBlock::kClockCount) {
+    // Grow from 0 to one-level table.
+    CHECK_EQ(size_, 0);
+    CHECK_EQ(tab_, 0);
+    CHECK_EQ(tab_idx_, 0);
+    size_ = nclk;
+    tab_idx_ = ctx->clock_alloc.Alloc(c);
+    tab_ = ctx->clock_alloc.Map(tab_idx_);
+    internal_memset(tab_, 0, sizeof(*tab_));
+    return;
+  }
+  // Growing two-level table.
+  if (size_ == 0) {
+    // Allocate first level table.
+    tab_idx_ = ctx->clock_alloc.Alloc(c);
+    tab_ = ctx->clock_alloc.Map(tab_idx_);
+    internal_memset(tab_, 0, sizeof(*tab_));
+  } else if (size_ <= ClockBlock::kClockCount) {
+    // Transform one-level table to two-level table.
+    u32 old = tab_idx_;
+    tab_idx_ = ctx->clock_alloc.Alloc(c);
+    tab_ = ctx->clock_alloc.Map(tab_idx_);
+    internal_memset(tab_, 0, sizeof(*tab_));
+    tab_->table[0] = old;
+  }
+  // At this point we have first level table allocated.
+  // Add second level tables as necessary.
+  for (uptr i = RoundUpTo(size_, ClockBlock::kClockCount);
+      i < nclk; i += ClockBlock::kClockCount) {
+    u32 idx = ctx->clock_alloc.Alloc(c);
+    ClockBlock *cb = ctx->clock_alloc.Map(idx);
+    internal_memset(cb, 0, sizeof(*cb));
+    CHECK_EQ(tab_->table[i/ClockBlock::kClockCount], 0);
+    tab_->table[i/ClockBlock::kClockCount] = idx;
+  }
+  size_ = nclk;
+}
+
 ClockElem &SyncClock::elem(unsigned tid) const {
   DCHECK_LT(tid, size_);
   if (size_ <= ClockBlock::kClockCount)
diff --git a/lib/tsan/rtl/tsan_clock.h b/lib/tsan/rtl/tsan_clock.h
index 4e352cb81d11..378b550fd11b 100644
--- a/lib/tsan/rtl/tsan_clock.h
+++ b/lib/tsan/rtl/tsan_clock.h
@@ -74,6 +74,7 @@ class SyncClock {
   u32 tab_idx_;
   u32 size_;
 
+  void ResetImpl();
   ClockElem &elem(unsigned tid) const;
 };
 
@@ -89,7 +90,7 @@ struct ThreadClock {
     return clk_[tid].epoch;
   }
 
-  void set(unsigned tid, u64 v);
+  void set(ClockCache *c, unsigned tid, u64 v);
 
   void set(u64 v) {
     DCHECK_GE(v, clk_[tid_].epoch);
@@ -108,6 +109,7 @@ struct ThreadClock {
   void release(ClockCache *c, SyncClock *dst) const;
   void acq_rel(ClockCache *c, SyncClock *dst);
   void ReleaseStore(ClockCache *c, SyncClock *dst) const;
+  void ResetCached(ClockCache *c);
 
   void DebugReset();
   void DebugDump(int(*printf)(const char *s, ...));
diff --git a/lib/tsan/rtl/tsan_dense_alloc.h b/lib/tsan/rtl/tsan_dense_alloc.h
index e9815c90a953..16dbdf391085 100644
--- a/lib/tsan/rtl/tsan_dense_alloc.h
+++ b/lib/tsan/rtl/tsan_dense_alloc.h
@@ -39,7 +39,7 @@ class DenseSlabAlloc {
   typedef DenseSlabAllocCache Cache;
   typedef typename Cache::IndexT IndexT;
 
-  DenseSlabAlloc() {
+  explicit DenseSlabAlloc(const char *name) {
     // Check that kL1Size and kL2Size are sane.
     CHECK_EQ(kL1Size & (kL1Size - 1), 0);
     CHECK_EQ(kL2Size & (kL2Size - 1), 0);
@@ -49,6 +49,7 @@ class DenseSlabAlloc {
     internal_memset(map_, 0, sizeof(map_));
     freelist_ = 0;
     fillpos_ = 0;
+    name_ = name;
   }
 
   ~DenseSlabAlloc() {
@@ -96,15 +97,19 @@ class DenseSlabAlloc {
   SpinMutex mtx_;
   IndexT freelist_;
   uptr fillpos_;
+  const char *name_;
 
   void Refill(Cache *c) {
     SpinMutexLock lock(&mtx_);
     if (freelist_ == 0) {
       if (fillpos_ == kL1Size) {
-        Printf("ThreadSanitizer: DenseSlabAllocator overflow. Dying.\n");
+        Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n",
+            name_, kL1Size, kL2Size);
         Die();
       }
-      T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), "DenseSlabAllocator");
+      VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n",
+          name_, fillpos_, kL1Size, kL2Size);
+      T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), name_);
       // Reserve 0 as invalid index.
       IndexT start = fillpos_ == 0 ? 1 : 0;
       for (IndexT i = start; i < kL2Size; i++) {
diff --git a/lib/tsan/rtl/tsan_interceptors.cc b/lib/tsan/rtl/tsan_interceptors.cc
index 334cc326daf6..001123f4941e 100644
--- a/lib/tsan/rtl/tsan_interceptors.cc
+++ b/lib/tsan/rtl/tsan_interceptors.cc
@@ -14,6 +14,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_linux.h"
 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
@@ -34,13 +35,11 @@
 using namespace __tsan;  // NOLINT
 
 #if SANITIZER_FREEBSD || SANITIZER_MAC
-#define __errno_location __error
 #define stdout __stdoutp
 #define stderr __stderrp
 #endif
 
 #if SANITIZER_ANDROID
-#define __errno_location __errno
 #define mallopt(a, b)
 #endif
 
@@ -84,7 +83,6 @@ DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size)
 DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
 extern "C" void *pthread_self();
 extern "C" void _exit(int status);
-extern "C" int *__errno_location();
 extern "C" int fileno_unlocked(void *stream);
 extern "C" int dirfd(void *dirp);
 #if !SANITIZER_FREEBSD && !SANITIZER_ANDROID
@@ -98,9 +96,6 @@ const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
 const int PTHREAD_MUTEX_RECURSIVE = 2;
 const int PTHREAD_MUTEX_RECURSIVE_NP = 2;
 #endif
-const int EINVAL = 22;
-const int EBUSY = 16;
-const int EOWNERDEAD = 130;
 #if !SANITIZER_FREEBSD && !SANITIZER_MAC
 const int EPOLL_CTL_ADD = 1;
 #endif
@@ -130,8 +125,6 @@ typedef long long_t;  // NOLINT
 # define F_TLOCK 2      /* Test and lock a region for exclusive use.  */
 # define F_TEST  3      /* Test a region for other processes locks.  */
 
-#define errno (*__errno_location())
-
 typedef void (*sighandler_t)(int sig);
 typedef void (*sigactionhandler_t)(int sig, my_siginfo_t *siginfo, void *uctx);
 
@@ -268,7 +261,7 @@ ScopedInterceptor::~ScopedInterceptor() {
 
 void ScopedInterceptor::EnableIgnores() {
   if (ignoring_) {
-    ThreadIgnoreBegin(thr_, pc_, false);
+    ThreadIgnoreBegin(thr_, pc_, /*save_stack=*/false);
     if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports++;
     if (in_ignored_lib_) {
       DCHECK(!thr_->in_ignored_lib);
@@ -466,8 +459,14 @@ static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) {
 static void LongJmp(ThreadState *thr, uptr *env) {
 #ifdef __powerpc__
   uptr mangled_sp = env[0];
-#elif SANITIZER_FREEBSD || SANITIZER_MAC
+#elif SANITIZER_FREEBSD
   uptr mangled_sp = env[2];
+#elif SANITIZER_MAC
+# ifdef __aarch64__
+    uptr mangled_sp = env[13];
+# else
+    uptr mangled_sp = env[2];
+# endif
 #elif defined(SANITIZER_LINUX)
 # ifdef __aarch64__
   uptr mangled_sp = env[13];
@@ -665,7 +664,7 @@ static bool fix_mmap_addr(void **addr, long_t sz, int flags) {
   if (*addr) {
     if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) {
       if (flags & MAP_FIXED) {
-        errno = EINVAL;
+        errno = errno_EINVAL;
         return false;
       } else {
         *addr = 0;
@@ -1122,7 +1121,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) {
 TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m);
   int res = REAL(pthread_mutex_destroy)(m);
-  if (res == 0 || res == EBUSY) {
+  if (res == 0 || res == errno_EBUSY) {
     MutexDestroy(thr, pc, (uptr)m);
   }
   return res;
@@ -1131,9 +1130,9 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
 TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
   int res = REAL(pthread_mutex_trylock)(m);
-  if (res == EOWNERDEAD)
+  if (res == errno_EOWNERDEAD)
     MutexRepair(thr, pc, (uptr)m);
-  if (res == 0 || res == EOWNERDEAD)
+  if (res == 0 || res == errno_EOWNERDEAD)
     MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
   return res;
 }
@@ -1311,7 +1310,7 @@ TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
 TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) {
   SCOPED_INTERCEPTOR_RAW(pthread_once, o, f);
   if (o == 0 || f == 0)
-    return EINVAL;
+    return errno_EINVAL;
   atomic_uint32_t *a;
   if (!SANITIZER_MAC)
     a = static_cast<atomic_uint32_t*>(o);
diff --git a/lib/tsan/rtl/tsan_interceptors_mac.cc b/lib/tsan/rtl/tsan_interceptors_mac.cc
index f6bf8a0e586b..4f1079467331 100644
--- a/lib/tsan/rtl/tsan_interceptors_mac.cc
+++ b/lib/tsan/rtl/tsan_interceptors_mac.cc
@@ -21,7 +21,10 @@
 #include "tsan_interface_ann.h"
 
 #include <libkern/OSAtomic.h>
+
+#if defined(__has_include) && __has_include(<xpc/xpc.h>)
 #include <xpc/xpc.h>
+#endif  // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
 
 typedef long long_t;  // NOLINT
 
@@ -235,6 +238,8 @@ TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
   REAL(os_lock_unlock)(lock);
 }
 
+#if defined(__has_include) && __has_include(<xpc/xpc.h>)
+
 TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
                  xpc_connection_t connection, xpc_handler_t handler) {
   SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
@@ -287,6 +292,8 @@ TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
   REAL(xpc_connection_cancel)(connection);
 }
 
+#endif  // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
+
 // On macOS, libc++ is always linked dynamically, so intercepting works the
 // usual way.
 #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
diff --git a/lib/tsan/rtl/tsan_interface_ann.cc b/lib/tsan/rtl/tsan_interface_ann.cc
index 45ec45bbdbbe..f68a0468de53 100644
--- a/lib/tsan/rtl/tsan_interface_ann.cc
+++ b/lib/tsan/rtl/tsan_interface_ann.cc
@@ -483,8 +483,8 @@ void __tsan_mutex_pre_lock(void *m, unsigned flagz) {
     else
       MutexPreLock(thr, pc, (uptr)m);
   }
-  ThreadIgnoreBegin(thr, pc, false);
-  ThreadIgnoreSyncBegin(thr, pc, false);
+  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
 }
 
 INTERFACE_ATTRIBUTE
@@ -510,8 +510,8 @@ int __tsan_mutex_pre_unlock(void *m, unsigned flagz) {
   } else {
     ret = MutexUnlock(thr, pc, (uptr)m, flagz);
   }
-  ThreadIgnoreBegin(thr, pc, false);
-  ThreadIgnoreSyncBegin(thr, pc, false);
+  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
   return ret;
 }
 
@@ -525,8 +525,8 @@ void __tsan_mutex_post_unlock(void *m, unsigned flagz) {
 INTERFACE_ATTRIBUTE
 void __tsan_mutex_pre_signal(void *addr, unsigned flagz) {
   SCOPED_ANNOTATION(__tsan_mutex_pre_signal);
-  ThreadIgnoreBegin(thr, pc, false);
-  ThreadIgnoreSyncBegin(thr, pc, false);
+  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
 }
 
 INTERFACE_ATTRIBUTE
@@ -547,7 +547,7 @@ void __tsan_mutex_pre_divert(void *addr, unsigned flagz) {
 INTERFACE_ATTRIBUTE
 void __tsan_mutex_post_divert(void *addr, unsigned flagz) {
   SCOPED_ANNOTATION(__tsan_mutex_post_divert);
-  ThreadIgnoreBegin(thr, pc, false);
-  ThreadIgnoreSyncBegin(thr, pc, false);
+  ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+  ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
 }
 }  // extern "C"
diff --git a/lib/tsan/rtl/tsan_interface_atomic.cc b/lib/tsan/rtl/tsan_interface_atomic.cc
index b22d5c1ecef8..d334394f5303 100644
--- a/lib/tsan/rtl/tsan_interface_atomic.cc
+++ b/lib/tsan/rtl/tsan_interface_atomic.cc
@@ -220,8 +220,7 @@ static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
 #endif
 
 template<typename T>
-static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
-    morder mo) {
+static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) {
   CHECK(IsLoadOrder(mo));
   // This fast-path is critical for performance.
   // Assume the access is atomic.
@@ -229,10 +228,17 @@ static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
     MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
     return NoTsanAtomicLoad(a, mo);
   }
-  SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, false);
-  AcquireImpl(thr, pc, &s->clock);
+  // Don't create sync object if it does not exist yet. For example, an atomic
+  // pointer is initialized to nullptr and then periodically acquire-loaded.
   T v = NoTsanAtomicLoad(a, mo);
-  s->mtx.ReadUnlock();
+  SyncVar *s = ctx->metamap.GetIfExistsAndLock((uptr)a, false);
+  if (s) {
+    AcquireImpl(thr, pc, &s->clock);
+    // Re-read under sync mutex because we need a consistent snapshot
+    // of the value and the clock we acquire.
+    v = NoTsanAtomicLoad(a, mo);
+    s->mtx.ReadUnlock();
+  }
   MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
   return v;
 }
diff --git a/lib/tsan/rtl/tsan_mman.cc b/lib/tsan/rtl/tsan_mman.cc
index 7169d5b02c04..1434cf688ce9 100644
--- a/lib/tsan/rtl/tsan_mman.cc
+++ b/lib/tsan/rtl/tsan_mman.cc
@@ -294,6 +294,8 @@ uptr __sanitizer_get_allocated_size(const void *p) {
 
 void __tsan_on_thread_idle() {
   ThreadState *thr = cur_thread();
+  thr->clock.ResetCached(&thr->proc()->clock_cache);
+  thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
   allocator()->SwallowCache(&thr->proc()->alloc_cache);
   internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache);
   ctx->metamap.OnProcIdle(thr->proc());
diff --git a/lib/tsan/rtl/tsan_platform.h b/lib/tsan/rtl/tsan_platform.h
index 60d9b9d8c452..bea1daba3952 100644
--- a/lib/tsan/rtl/tsan_platform.h
+++ b/lib/tsan/rtl/tsan_platform.h
@@ -100,6 +100,37 @@ struct Mapping {
 };
 
 #define TSAN_MID_APP_RANGE 1
+#elif defined(__aarch64__) && defined(__APPLE__)
+/*
+C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
+0000 0000 00 - 0100 0000 00: -                                    (4 GB)
+0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks  (4 GB)
+0200 0000 00 - 0300 0000 00: heap                                 (4 GB)
+0300 0000 00 - 0400 0000 00: -                                    (4 GB)
+0400 0000 00 - 0c00 0000 00: shadow memory                       (32 GB)
+0c00 0000 00 - 0d00 0000 00: -                                    (4 GB)
+0d00 0000 00 - 0e00 0000 00: metainfo                             (4 GB)
+0e00 0000 00 - 0f00 0000 00: -                                    (4 GB)
+0f00 0000 00 - 1000 0000 00: traces                               (4 GB)
+*/
+struct Mapping {
+  static const uptr kLoAppMemBeg   = 0x0100000000ull;
+  static const uptr kLoAppMemEnd   = 0x0200000000ull;
+  static const uptr kHeapMemBeg    = 0x0200000000ull;
+  static const uptr kHeapMemEnd    = 0x0300000000ull;
+  static const uptr kShadowBeg     = 0x0400000000ull;
+  static const uptr kShadowEnd     = 0x0c00000000ull;
+  static const uptr kMetaShadowBeg = 0x0d00000000ull;
+  static const uptr kMetaShadowEnd = 0x0e00000000ull;
+  static const uptr kTraceMemBeg   = 0x0f00000000ull;
+  static const uptr kTraceMemEnd   = 0x1000000000ull;
+  static const uptr kHiAppMemBeg   = 0x1000000000ull;
+  static const uptr kHiAppMemEnd   = 0x1000000000ull;
+  static const uptr kAppMemMsk     =          0x0ull;
+  static const uptr kAppMemXor     =          0x0ull;
+  static const uptr kVdsoBeg       = 0x7000000000000000ull;
+};
+
 #elif defined(__aarch64__)
 // AArch64 supports multiple VMA which leads to multiple address transformation
 // functions.  To support these multiple VMAS transformations and mappings TSAN
@@ -389,7 +420,7 @@ uptr MappingImpl(void) {
 
 template<int Type>
 uptr MappingArchImpl(void) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return MappingImpl<Mapping39, Type>();
     case 42: return MappingImpl<Mapping42, Type>();
@@ -542,7 +573,7 @@ bool IsAppMemImpl(uptr mem) {
 
 ALWAYS_INLINE
 bool IsAppMem(uptr mem) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return IsAppMemImpl<Mapping39>(mem);
     case 42: return IsAppMemImpl<Mapping42>(mem);
@@ -569,7 +600,7 @@ bool IsShadowMemImpl(uptr mem) {
 
 ALWAYS_INLINE
 bool IsShadowMem(uptr mem) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return IsShadowMemImpl<Mapping39>(mem);
     case 42: return IsShadowMemImpl<Mapping42>(mem);
@@ -596,7 +627,7 @@ bool IsMetaMemImpl(uptr mem) {
 
 ALWAYS_INLINE
 bool IsMetaMem(uptr mem) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return IsMetaMemImpl<Mapping39>(mem);
     case 42: return IsMetaMemImpl<Mapping42>(mem);
@@ -633,7 +664,7 @@ uptr MemToShadowImpl(uptr x) {
 
 ALWAYS_INLINE
 uptr MemToShadow(uptr x) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return MemToShadowImpl<Mapping39>(x);
     case 42: return MemToShadowImpl<Mapping42>(x);
@@ -672,7 +703,7 @@ u32 *MemToMetaImpl(uptr x) {
 
 ALWAYS_INLINE
 u32 *MemToMeta(uptr x) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return MemToMetaImpl<Mapping39>(x);
     case 42: return MemToMetaImpl<Mapping42>(x);
@@ -724,7 +755,7 @@ uptr ShadowToMemImpl(uptr s) {
 
 ALWAYS_INLINE
 uptr ShadowToMem(uptr s) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return ShadowToMemImpl<Mapping39>(s);
     case 42: return ShadowToMemImpl<Mapping42>(s);
@@ -759,7 +790,7 @@ uptr GetThreadTraceImpl(int tid) {
 
 ALWAYS_INLINE
 uptr GetThreadTrace(int tid) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return GetThreadTraceImpl<Mapping39>(tid);
     case 42: return GetThreadTraceImpl<Mapping42>(tid);
@@ -789,7 +820,7 @@ uptr GetThreadTraceHeaderImpl(int tid) {
 
 ALWAYS_INLINE
 uptr GetThreadTraceHeader(int tid) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
   switch (vmaSize) {
     case 39: return GetThreadTraceHeaderImpl<Mapping39>(tid);
     case 42: return GetThreadTraceHeaderImpl<Mapping42>(tid);
diff --git a/lib/tsan/rtl/tsan_platform_linux.cc b/lib/tsan/rtl/tsan_platform_linux.cc
index d05c0e701e72..0ba01babe69a 100644
--- a/lib/tsan/rtl/tsan_platform_linux.cc
+++ b/lib/tsan/rtl/tsan_platform_linux.cc
@@ -47,7 +47,6 @@
 #include <sys/resource.h>
 #include <sys/stat.h>
 #include <unistd.h>
-#include <errno.h>
 #include <sched.h>
 #include <dlfcn.h>
 #if SANITIZER_LINUX
@@ -182,17 +181,15 @@ static void MapRodata() {
   }
   // Map the file into shadow of .rodata sections.
   MemoryMappingLayout proc_maps(/*cache_enabled*/true);
-  uptr start, end, offset, prot;
   // Reusing the buffer 'name'.
-  while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), &prot)) {
-    if (name[0] != 0 && name[0] != '['
-        && (prot & MemoryMappingLayout::kProtectionRead)
-        && (prot & MemoryMappingLayout::kProtectionExecute)
-        && !(prot & MemoryMappingLayout::kProtectionWrite)
-        && IsAppMem(start)) {
+  MemoryMappedSegment segment(name, ARRAY_SIZE(name));
+  while (proc_maps.Next(&segment)) {
+    if (segment.filename[0] != 0 && segment.filename[0] != '[' &&
+        segment.IsReadable() && segment.IsExecutable() &&
+        !segment.IsWritable() && IsAppMem(segment.start)) {
       // Assume it's .rodata
-      char *shadow_start = (char*)MemToShadow(start);
-      char *shadow_end = (char*)MemToShadow(end);
+      char *shadow_start = (char *)MemToShadow(segment.start);
+      char *shadow_end = (char *)MemToShadow(segment.end);
       for (char *p = shadow_start; p < shadow_end; p += marker.size()) {
         internal_mmap(p, Min<uptr>(marker.size(), shadow_end - p),
                       PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0);
diff --git a/lib/tsan/rtl/tsan_platform_mac.cc b/lib/tsan/rtl/tsan_platform_mac.cc
index a82bcd01bbf4..73a656ffca5e 100644
--- a/lib/tsan/rtl/tsan_platform_mac.cc
+++ b/lib/tsan/rtl/tsan_platform_mac.cc
@@ -230,6 +230,14 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread,
 #endif
 
 void InitializePlatformEarly() {
+#if defined(__aarch64__)
+  uptr max_vm = GetMaxVirtualAddress() + 1;
+  if (max_vm != Mapping::kHiAppMemEnd) {
+    Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
+           max_vm, Mapping::kHiAppMemEnd);
+    Die();
+  }
+#endif
 }
 
 void InitializePlatform() {
diff --git a/lib/tsan/rtl/tsan_platform_posix.cc b/lib/tsan/rtl/tsan_platform_posix.cc
index 0732c83d689d..e4f90a811c35 100644
--- a/lib/tsan/rtl/tsan_platform_posix.cc
+++ b/lib/tsan/rtl/tsan_platform_posix.cc
@@ -46,6 +46,9 @@ void InitializeShadowMemory() {
 #elif defined(__mips64)
   const uptr kMadviseRangeBeg  = 0xff00000000ull;
   const uptr kMadviseRangeSize = 0x0100000000ull;
+#elif defined(__aarch64__) && defined(__APPLE__)
+  uptr kMadviseRangeBeg = LoAppMemBeg();
+  uptr kMadviseRangeSize = LoAppMemEnd() - LoAppMemBeg();
 #elif defined(__aarch64__)
   uptr kMadviseRangeBeg = 0;
   uptr kMadviseRangeSize = 0;
@@ -115,21 +118,24 @@ static void ProtectRange(uptr beg, uptr end) {
 void CheckAndProtect() {
   // Ensure that the binary is indeed compiled with -pie.
   MemoryMappingLayout proc_maps(true);
-  uptr p, end, prot;
-  while (proc_maps.Next(&p, &end, 0, 0, 0, &prot)) {
-    if (IsAppMem(p))
+  MemoryMappedSegment segment;
+  while (proc_maps.Next(&segment)) {
+    if (IsAppMem(segment.start)) continue;
+    if (segment.start >= HeapMemEnd() && segment.start < HeapEnd()) continue;
+    if (segment.protection == 0)  // Zero page or mprotected.
       continue;
-    if (p >= HeapMemEnd() &&
-        p < HeapEnd())
-      continue;
-    if (prot == 0)  // Zero page or mprotected.
-      continue;
-    if (p >= VdsoBeg())  // vdso
+    if (segment.start >= VdsoBeg())  // vdso
       break;
-    Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n", p, end);
+    Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n",
+           segment.start, segment.end);
     Die();
   }
 
+#if defined(__aarch64__) && defined(__APPLE__)
+  ProtectRange(HeapMemEnd(), ShadowBeg());
+  ProtectRange(ShadowEnd(), MetaShadowBeg());
+  ProtectRange(MetaShadowEnd(), TraceMemBeg());
+#else
   ProtectRange(LoAppMemEnd(), ShadowBeg());
   ProtectRange(ShadowEnd(), MetaShadowBeg());
 #ifdef TSAN_MID_APP_RANGE
@@ -143,6 +149,7 @@ void CheckAndProtect() {
   ProtectRange(TraceMemBeg(), TraceMemEnd());
   ProtectRange(TraceMemEnd(), HeapMemBeg());
   ProtectRange(HeapEnd(), HiAppMemBeg());
+#endif
 }
 #endif
 
diff --git a/lib/tsan/rtl/tsan_rtl.cc b/lib/tsan/rtl/tsan_rtl.cc
index fa60f3247c38..a01525302b02 100644
--- a/lib/tsan/rtl/tsan_rtl.cc
+++ b/lib/tsan/rtl/tsan_rtl.cc
@@ -104,7 +104,8 @@ Context::Context()
   , racy_stacks(MBlockRacyStacks)
   , racy_addresses(MBlockRacyAddresses)
   , fired_suppressions_mtx(MutexTypeFired, StatMtxFired)
-  , fired_suppressions(8) {
+  , fired_suppressions(8)
+  , clock_alloc("clock allocator") {
 }
 
 // The objects are allocated in TLS, so one may rely on zero-initialization.
diff --git a/lib/tsan/rtl/tsan_rtl_aarch64.S b/lib/tsan/rtl/tsan_rtl_aarch64.S
index ef06f0444ae4..61171d635c18 100644
--- a/lib/tsan/rtl/tsan_rtl_aarch64.S
+++ b/lib/tsan/rtl/tsan_rtl_aarch64.S
@@ -1,13 +1,46 @@
+// The content of this file is AArch64-only:
+#if defined(__aarch64__)
+
 #include "sanitizer_common/sanitizer_asm.h"
 
+#if !defined(__APPLE__)
 .section .bss
 .type	__tsan_pointer_chk_guard, %object
-.size	__tsan_pointer_chk_guard, 8
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__tsan_pointer_chk_guard))
 __tsan_pointer_chk_guard:
 .zero	8
+#endif
+
+#if defined(__APPLE__)
+.align  2
+
+.section  __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long _setjmp$non_lazy_ptr
+_setjmp$non_lazy_ptr:
+.indirect_symbol _setjmp
+.long 0
+
+.section  __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long __setjmp$non_lazy_ptr
+__setjmp$non_lazy_ptr:
+.indirect_symbol __setjmp
+.long 0
+
+.section  __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long _sigsetjmp$non_lazy_ptr
+_sigsetjmp$non_lazy_ptr:
+.indirect_symbol _sigsetjmp
+.long 0
+#endif
 
+#if !defined(__APPLE__)
 .section .text
+#else
+.section __TEXT,__text
+.align 3
+#endif
 
+#if !defined(__APPLE__)
 // GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp
 // functions) by XORing them with a random guard pointer.  For AArch64 it is a
 // global variable rather than a TCB one (as for x86_64/powerpc) and althought
@@ -16,9 +49,9 @@ __tsan_pointer_chk_guard:
 // not stable). So InitializeGuardPtr obtains the pointer guard value by
 // issuing a setjmp and checking the resulting pointers values against the
 // original ones.
-.hidden _Z18InitializeGuardPtrv
+ASM_HIDDEN(_Z18InitializeGuardPtrv)
 .global _Z18InitializeGuardPtrv
-.type _Z18InitializeGuardPtrv, @function
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv))
 _Z18InitializeGuardPtrv:
   CFI_STARTPROC
   // Allocates a jmp_buf for the setjmp call.
@@ -55,12 +88,14 @@ _Z18InitializeGuardPtrv:
   CFI_DEF_CFA (31, 0)
   ret
   CFI_ENDPROC
-.size _Z18InitializeGuardPtrv, .-_Z18InitializeGuardPtrv
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv))
+#endif
 
-.hidden __tsan_setjmp
+ASM_HIDDEN(__tsan_setjmp)
 .comm _ZN14__interception11real_setjmpE,8,8
-.type setjmp, @function
-setjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp):
   CFI_STARTPROC
 
   // save env parameters for function call
@@ -78,14 +113,19 @@ setjmp:
   CFI_OFFSET (19, -16)
   mov     x19, x0
 
+#if !defined(__APPLE__)
   // SP pointer mangling (see glibc setjmp)
   adrp    x2, __tsan_pointer_chk_guard
   ldr     x2, [x2, #:lo12:__tsan_pointer_chk_guard]
   add     x0, x29, 32
   eor     x1, x2, x0
+#else
+  add     x0, x29, 32
+  mov     x1, x0
+#endif
 
   // call tsan interceptor
-  bl      __tsan_setjmp
+  bl      ASM_TSAN_SYMBOL(__tsan_setjmp)
 
   // restore env parameter
   mov     x0, x19
@@ -96,18 +136,24 @@ setjmp:
   CFI_DEF_CFA (31, 0)
 
   // tail jump to libc setjmp
+#if !defined(__APPLE__)
   adrp    x1, :got:_ZN14__interception11real_setjmpE
   ldr     x1, [x1, #:got_lo12:_ZN14__interception11real_setjmpE]
   ldr     x1, [x1]
+#else
+  adrp    x1, _setjmp$non_lazy_ptr@page
+  add     x1, x1, _setjmp$non_lazy_ptr@pageoff
+  ldr     x1, [x1]
+#endif
   br      x1
 
   CFI_ENDPROC
-.size setjmp, .-setjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp))
 
 .comm _ZN14__interception12real__setjmpE,8,8
-.globl _setjmp
-.type _setjmp, @function
-_setjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp):
   CFI_STARTPROC
 
   // save env parameters for function call
@@ -125,14 +171,19 @@ _setjmp:
   CFI_OFFSET (19, -16)
   mov     x19, x0
 
+#if !defined(__APPLE__)
   // SP pointer mangling (see glibc setjmp)
   adrp    x2, __tsan_pointer_chk_guard
   ldr     x2, [x2, #:lo12:__tsan_pointer_chk_guard]
   add     x0, x29, 32
   eor     x1, x2, x0
+#else
+  add     x0, x29, 32
+  mov     x1, x0
+#endif
 
   // call tsan interceptor
-  bl      __tsan_setjmp
+  bl      ASM_TSAN_SYMBOL(__tsan_setjmp)
 
   // Restore jmp_buf parameter
   mov     x0, x19
@@ -143,18 +194,24 @@ _setjmp:
   CFI_DEF_CFA (31, 0)
 
   // tail jump to libc setjmp
+#if !defined(__APPLE__)
   adrp    x1, :got:_ZN14__interception12real__setjmpE
   ldr     x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE]
   ldr     x1, [x1]
+#else
+  adrp    x1, __setjmp$non_lazy_ptr@page
+  add     x1, x1, __setjmp$non_lazy_ptr@pageoff
+  ldr     x1, [x1]
+#endif
   br      x1
 
   CFI_ENDPROC
-.size _setjmp, .-_setjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp))
 
 .comm _ZN14__interception14real_sigsetjmpE,8,8
-.globl sigsetjmp
-.type sigsetjmp, @function
-sigsetjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp):
   CFI_STARTPROC
 
   // save env parameters for function call
@@ -174,14 +231,19 @@ sigsetjmp:
   mov     w20, w1
   mov     x19, x0
 
+#if !defined(__APPLE__)
   // SP pointer mangling (see glibc setjmp)
   adrp    x2, __tsan_pointer_chk_guard
   ldr     x2, [x2, #:lo12:__tsan_pointer_chk_guard]
   add     x0, x29, 32
   eor     x1, x2, x0
+#else
+  add     x0, x29, 32
+  mov     x1, x0
+#endif
 
   // call tsan interceptor
-  bl      __tsan_setjmp
+  bl      ASM_TSAN_SYMBOL(__tsan_setjmp)
 
   // restore env parameter
   mov     w1, w20
@@ -195,17 +257,24 @@ sigsetjmp:
   CFI_DEF_CFA (31, 0)
 
   // tail jump to libc sigsetjmp
+#if !defined(__APPLE__)
   adrp    x2, :got:_ZN14__interception14real_sigsetjmpE
   ldr     x2, [x2, #:got_lo12:_ZN14__interception14real_sigsetjmpE]
   ldr     x2, [x2]
+#else
+  adrp    x2, _sigsetjmp$non_lazy_ptr@page
+  add     x2, x2, _sigsetjmp$non_lazy_ptr@pageoff
+  ldr     x2, [x2]
+#endif
   br      x2
   CFI_ENDPROC
-.size sigsetjmp, .-sigsetjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp))
 
+#if !defined(__APPLE__)
 .comm _ZN14__interception16real___sigsetjmpE,8,8
-.globl __sigsetjmp
-.type __sigsetjmp, @function
-__sigsetjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp):
   CFI_STARTPROC
 
   // save env parameters for function call
@@ -225,14 +294,16 @@ __sigsetjmp:
   mov     w20, w1
   mov     x19, x0
 
+#if !defined(__APPLE__)
   // SP pointer mangling (see glibc setjmp)
   adrp    x2, __tsan_pointer_chk_guard
   ldr     x2, [x2, #:lo12:__tsan_pointer_chk_guard]
   add     x0, x29, 32
   eor     x1, x2, x0
+#endif
 
   // call tsan interceptor
-  bl      __tsan_setjmp
+  bl      ASM_TSAN_SYMBOL(__tsan_setjmp)
 
   mov     w1, w20
   mov     x0, x19
@@ -245,14 +316,22 @@ __sigsetjmp:
   CFI_DEF_CFA (31, 0)
 
   // tail jump to libc __sigsetjmp
+#if !defined(__APPLE__)
   adrp    x2, :got:_ZN14__interception16real___sigsetjmpE
   ldr     x2, [x2, #:got_lo12:_ZN14__interception16real___sigsetjmpE]
   ldr     x2, [x2]
+#else
+  adrp    x2, ASM_TSAN_SYMBOL(__sigsetjmp)@page
+  add     x2, x2, ASM_TSAN_SYMBOL(__sigsetjmp)@pageoff
+#endif
   br      x2
   CFI_ENDPROC
-.size __sigsetjmp, .-__sigsetjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
+#endif
 
 #if defined(__linux__)
 /* We do not need executable stack.  */
 .section        .note.GNU-stack,"",@progbits
 #endif
+
+#endif
diff --git a/lib/tsan/rtl/tsan_rtl_amd64.S b/lib/tsan/rtl/tsan_rtl_amd64.S
index caa832375e52..98947fd2a1ba 100644
--- a/lib/tsan/rtl/tsan_rtl_amd64.S
+++ b/lib/tsan/rtl/tsan_rtl_amd64.S
@@ -1,4 +1,8 @@
+// The content of this file is x86_64-only:
+#if defined(__x86_64__)
+
 #include "sanitizer_common/sanitizer_asm.h"
+
 #if !defined(__APPLE__)
 .section .text
 #else
@@ -357,3 +361,5 @@ ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
 /* We do not need executable stack.  */
 .section        .note.GNU-stack,"",@progbits
 #endif
+
+#endif
diff --git a/lib/tsan/rtl/tsan_rtl_mutex.cc b/lib/tsan/rtl/tsan_rtl_mutex.cc
index 54938f37e243..2f85811620f1 100644
--- a/lib/tsan/rtl/tsan_rtl_mutex.cc
+++ b/lib/tsan/rtl/tsan_rtl_mutex.cc
@@ -413,10 +413,10 @@ void Acquire(ThreadState *thr, uptr pc, uptr addr) {
 static void UpdateClockCallback(ThreadContextBase *tctx_base, void *arg) {
   ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
   ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+  u64 epoch = tctx->epoch1;
   if (tctx->status == ThreadStatusRunning)
-    thr->clock.set(tctx->tid, tctx->thr->fast_state.epoch());
-  else
-    thr->clock.set(tctx->tid, tctx->epoch1);
+    epoch = tctx->thr->fast_state.epoch();
+  thr->clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
 }
 
 void AcquireGlobal(ThreadState *thr, uptr pc) {
@@ -456,10 +456,10 @@ void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) {
 static void UpdateSleepClockCallback(ThreadContextBase *tctx_base, void *arg) {
   ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
   ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+  u64 epoch = tctx->epoch1;
   if (tctx->status == ThreadStatusRunning)
-    thr->last_sleep_clock.set(tctx->tid, tctx->thr->fast_state.epoch());
-  else
-    thr->last_sleep_clock.set(tctx->tid, tctx->epoch1);
+    epoch = tctx->thr->fast_state.epoch();
+  thr->last_sleep_clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
 }
 
 void AfterSleep(ThreadState *thr, uptr pc) {
diff --git a/lib/tsan/rtl/tsan_rtl_report.cc b/lib/tsan/rtl/tsan_rtl_report.cc
index 68b9f50308ea..85a982941ed7 100644
--- a/lib/tsan/rtl/tsan_rtl_report.cc
+++ b/lib/tsan/rtl/tsan_rtl_report.cc
@@ -314,7 +314,7 @@ void ScopedReport::AddLocation(uptr addr, uptr size) {
     return;
 #if !SANITIZER_GO
   int fd = -1;
-  int creat_tid = -1;
+  int creat_tid = kInvalidTid;
   u32 creat_stack = 0;
   if (FdLocation(addr, &fd, &creat_tid, &creat_stack)) {
     ReportLocation *loc = ReportLocation::New(ReportLocationFD);
diff --git a/lib/tsan/rtl/tsan_rtl_thread.cc b/lib/tsan/rtl/tsan_rtl_thread.cc
index 67eebf5d0c38..83fab082afe3 100644
--- a/lib/tsan/rtl/tsan_rtl_thread.cc
+++ b/lib/tsan/rtl/tsan_rtl_thread.cc
@@ -142,6 +142,10 @@ void ThreadContext::OnFinished() {
 
   if (common_flags()->detect_deadlocks)
     ctx->dd->DestroyLogicalThread(thr->dd_lt);
+  thr->clock.ResetCached(&thr->proc()->clock_cache);
+#if !SANITIZER_GO
+  thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
+#endif
   thr->~ThreadState();
 #if TSAN_COLLECT_STATS
   StatAggregate(ctx->stat, thr->stat);
diff --git a/lib/tsan/rtl/tsan_stat.cc b/lib/tsan/rtl/tsan_stat.cc
index 2ee688bf5771..18c83d5c6dac 100644
--- a/lib/tsan/rtl/tsan_stat.cc
+++ b/lib/tsan/rtl/tsan_stat.cc
@@ -75,14 +75,11 @@ void StatOutput(u64 *stat) {
   name[StatClockAcquire]                 = "Clock acquire                     ";
   name[StatClockAcquireEmpty]            = "  empty clock                     ";
   name[StatClockAcquireFastRelease]      = "  fast from release-store         ";
-  name[StatClockAcquireLarge]            = "  contains my tid                 ";
-  name[StatClockAcquireRepeat]           = "  repeated (fast)                 ";
   name[StatClockAcquireFull]             = "  full (slow)                     ";
   name[StatClockAcquiredSomething]       = "  acquired something              ";
   name[StatClockRelease]                 = "Clock release                     ";
   name[StatClockReleaseResize]           = "  resize                          ";
-  name[StatClockReleaseFast1]            = "  fast1                           ";
-  name[StatClockReleaseFast2]            = "  fast2                           ";
+  name[StatClockReleaseFast]             = "  fast                            ";
   name[StatClockReleaseSlow]             = "  dirty overflow (slow)           ";
   name[StatClockReleaseFull]             = "  full (slow)                     ";
   name[StatClockReleaseAcquired]         = "  was acquired                    ";
diff --git a/lib/tsan/rtl/tsan_stat.h b/lib/tsan/rtl/tsan_stat.h
index 7d2791ebbfcc..42d6a2b63532 100644
--- a/lib/tsan/rtl/tsan_stat.h
+++ b/lib/tsan/rtl/tsan_stat.h
@@ -74,15 +74,12 @@ enum StatType {
   StatClockAcquire,
   StatClockAcquireEmpty,
   StatClockAcquireFastRelease,
-  StatClockAcquireLarge,
-  StatClockAcquireRepeat,
   StatClockAcquireFull,
   StatClockAcquiredSomething,
   // Clocks - release.
   StatClockRelease,
   StatClockReleaseResize,
-  StatClockReleaseFast1,
-  StatClockReleaseFast2,
+  StatClockReleaseFast,
   StatClockReleaseSlow,
   StatClockReleaseFull,
   StatClockReleaseAcquired,
diff --git a/lib/tsan/rtl/tsan_sync.cc b/lib/tsan/rtl/tsan_sync.cc
index 4cc3cb89c34f..44ae558fa1b2 100644
--- a/lib/tsan/rtl/tsan_sync.cc
+++ b/lib/tsan/rtl/tsan_sync.cc
@@ -53,7 +53,9 @@ void SyncVar::Reset(Processor *proc) {
   }
 }
 
-MetaMap::MetaMap() {
+MetaMap::MetaMap()
+    : block_alloc_("heap block allocator")
+    , sync_alloc_("sync allocator") {
   atomic_store(&uid_gen_, 0, memory_order_relaxed);
 }
 
diff --git a/lib/tsan/tests/CMakeLists.txt b/lib/tsan/tests/CMakeLists.txt
index ca43a928d1b8..f8aec6854bb3 100644
--- a/lib/tsan/tests/CMakeLists.txt
+++ b/lib/tsan/tests/CMakeLists.txt
@@ -15,6 +15,7 @@ set(TSAN_UNITTEST_CFLAGS
 
 if(APPLE)
   list(APPEND TSAN_UNITTEST_CFLAGS ${DARWIN_osx_CFLAGS})
+  list(APPEND TSAN_UNITTEST_LINKFLAGS ${DARWIN_osx_LINKFLAGS})
 endif()
 
 set(TSAN_RTL_HEADERS)
diff --git a/lib/tsan/tests/unit/tsan_clock_test.cc b/lib/tsan/tests/unit/tsan_clock_test.cc
index 83e25fb5a933..73104dd6b9d4 100644
--- a/lib/tsan/tests/unit/tsan_clock_test.cc
+++ b/lib/tsan/tests/unit/tsan_clock_test.cc
@@ -26,13 +26,13 @@ TEST(Clock, VectorBasic) {
   clk.tick();
   ASSERT_EQ(clk.size(), 1U);
   ASSERT_EQ(clk.get(0), 1U);
-  clk.set(3, clk.get(3) + 1);
+  clk.set(&cache, 3, clk.get(3) + 1);
   ASSERT_EQ(clk.size(), 4U);
   ASSERT_EQ(clk.get(0), 1U);
   ASSERT_EQ(clk.get(1), 0U);
   ASSERT_EQ(clk.get(2), 0U);
   ASSERT_EQ(clk.get(3), 1U);
-  clk.set(3, clk.get(3) + 1);
+  clk.set(&cache, 3, clk.get(3) + 1);
   ASSERT_EQ(clk.get(3), 2U);
 }
 
@@ -86,24 +86,26 @@ TEST(Clock, RepeatedAcquire) {
 
 TEST(Clock, ManyThreads) {
   SyncClock chunked;
-  for (unsigned i = 0; i < 100; i++) {
+  for (unsigned i = 0; i < 200; i++) {
     ThreadClock vector(0);
     vector.tick();
-    vector.set(i, 1);
+    vector.set(&cache, i, i + 1);
     vector.release(&cache, &chunked);
     ASSERT_EQ(i + 1, chunked.size());
     vector.acquire(&cache, &chunked);
     ASSERT_EQ(i + 1, vector.size());
   }
 
-  for (unsigned i = 0; i < 100; i++)
-    ASSERT_EQ(1U, chunked.get(i));
+  for (unsigned i = 0; i < 200; i++) {
+    printf("i=%d\n", i);
+    ASSERT_EQ(i + 1, chunked.get(i));
+  }
 
   ThreadClock vector(1);
   vector.acquire(&cache, &chunked);
-  ASSERT_EQ(100U, vector.size());
-  for (unsigned i = 0; i < 100; i++)
-    ASSERT_EQ(1U, vector.get(i));
+  ASSERT_EQ(200U, vector.size());
+  for (unsigned i = 0; i < 200; i++)
+    ASSERT_EQ(i + 1, vector.get(i));
 
   chunked.Reset(&cache);
 }
@@ -151,7 +153,7 @@ TEST(Clock, Growth) {
   {
     ThreadClock vector(10);
     vector.tick();
-    vector.set(5, 42);
+    vector.set(&cache, 5, 42);
     SyncClock sync;
     vector.release(&cache, &sync);
     ASSERT_EQ(sync.size(), 11U);
@@ -180,8 +182,8 @@ TEST(Clock, Growth) {
   {
     ThreadClock vector(100);
     vector.tick();
-    vector.set(5, 42);
-    vector.set(90, 84);
+    vector.set(&cache, 5, 42);
+    vector.set(&cache, 90, 84);
     SyncClock sync;
     vector.release(&cache, &sync);
     ASSERT_EQ(sync.size(), 101U);
@@ -212,6 +214,42 @@ TEST(Clock, Growth) {
   }
 }
 
+TEST(Clock, Growth2) {
+  // Test clock growth for every pair of sizes:
+  const uptr sizes[] = {0, 1, 2, 30, 61, 62, 63, 64, 65, 66, 100, 124, 125, 126,
+      127, 128, 129, 130, 188, 189, 190, 191, 192, 193, 254, 255};
+  const uptr n = sizeof(sizes) / sizeof(sizes[0]);
+  for (uptr fi = 0; fi < n; fi++) {
+    for (uptr ti = fi + 1; ti < n; ti++) {
+      const uptr from = sizes[fi];
+      const uptr to = sizes[ti];
+      SyncClock sync;
+      ThreadClock vector(0);
+      for (uptr i = 0; i < from; i++)
+        vector.set(&cache, i, i + 1);
+      if (from != 0)
+        vector.release(&cache, &sync);
+      ASSERT_EQ(sync.size(), from);
+      for (uptr i = 0; i < from; i++)
+        ASSERT_EQ(sync.get(i), i + 1);
+      for (uptr i = 0; i < to; i++)
+        vector.set(&cache, i, i + 1);
+      vector.release(&cache, &sync);
+      ASSERT_EQ(sync.size(), to);
+      for (uptr i = 0; i < to; i++)
+        ASSERT_EQ(sync.get(i), i + 1);
+      vector.set(&cache, to + 1, to + 1);
+      vector.release(&cache, &sync);
+      ASSERT_EQ(sync.size(), to + 2);
+      for (uptr i = 0; i < to; i++)
+        ASSERT_EQ(sync.get(i), i + 1);
+      ASSERT_EQ(sync.get(to), 0U);
+      ASSERT_EQ(sync.get(to + 1), to + 1);
+      sync.Reset(&cache);
+    }
+  }
+}
+
 const uptr kThreads = 4;
 const uptr kClocks = 4;
 
diff --git a/test/asan/CMakeLists.txt b/test/asan/CMakeLists.txt
index 87fa9d138748..8bfc15b5c6f6 100644
--- a/test/asan/CMakeLists.txt
+++ b/test/asan/CMakeLists.txt
@@ -84,12 +84,13 @@ foreach(arch ${ASAN_TEST_ARCH})
 endforeach()
 
 # iOS and iOS simulator test suites
-# These are not added into "check-all", in order to run these tests, you have to
-# manually call (from the build directory).  They also require that an extra env
+# These are not added into "check-all", in order to run these tests, use
+# "check-asan-iossim-x86_64" and similar. They also require that an extra env
 # variable to select which iOS device or simulator to use, e.g.:
-# $ SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER=BBE44C1C-8AAA-4000-8D06-91C89ED58172
-# $ ./bin/llvm-lit ./projects/compiler-rt/test/asan/IOSSimI386Config
+# SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER="iPhone 6"
 if(APPLE)
+  set(EXCLUDE_FROM_ALL ON)
+
   set(ASAN_TEST_TARGET_CC ${COMPILER_RT_TEST_COMPILER})
   set(ASAN_TEST_IOS "1")
   pythonize_bool(ASAN_TEST_IOS)
@@ -108,6 +109,9 @@ if(APPLE)
       ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
       ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg
       )
+    add_lit_testsuite(check-asan-iossim-${arch} "AddressSanitizer iOS Simulator ${arch} tests"
+      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/
+      DEPENDS ${ASAN_TEST_DEPS})
   endforeach()
 
   foreach (arch ${DARWIN_ios_ARCHS})
@@ -123,7 +127,12 @@ if(APPLE)
       ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
       ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg
       )
+    add_lit_testsuite(check-asan-ios-${arch} "AddressSanitizer iOS ${arch} tests"
+      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/
+      DEPENDS ${ASAN_TEST_DEPS})
   endforeach()
+
+  set(EXCLUDE_FROM_ALL OFF)
 endif()
 
 # Add unit tests.
diff --git a/test/asan/TestCases/Posix/allow_user_segv.cc b/test/asan/TestCases/Posix/allow_user_segv.cc
index fee58943074e..4bec6ad89609 100644
--- a/test/asan/TestCases/Posix/allow_user_segv.cc
+++ b/test/asan/TestCases/Posix/allow_user_segv.cc
@@ -1,22 +1,21 @@
 // Regression test for
 // https://code.google.com/p/address-sanitizer/issues/detail?id=180
 
-// RUN: %clangxx_asan -O0 %s -o %t && %env_asan_opts=handle_segv=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK0
-// RUN: %clangxx_asan -O2 %s -o %t && %env_asan_opts=handle_segv=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK0
+// clang-format off
+// RUN: %clangxx_asan -O0 %s -o %t
 
-// RUN: %clangxx_asan -O0 %s -o %t && %env_asan_opts=handle_segv=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK1
-// RUN: %clangxx_asan -O2 %s -o %t && %env_asan_opts=handle_segv=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK1
+// RUN: %env_asan_opts=handle_segv=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK0
+// RUN: %env_asan_opts=handle_segv=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK1
+// RUN: %env_asan_opts=handle_segv=2 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK2
 
-// RUN: %clangxx_asan -O0 %s -o %t && %env_asan_opts=handle_segv=2 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK2
-// RUN: %clangxx_asan -O2 %s -o %t && %env_asan_opts=handle_segv=2 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK2
+// RUN: %env_asan_opts=handle_segv=0:allow_user_segv_handler=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK0
+// RUN: %env_asan_opts=handle_segv=1:allow_user_segv_handler=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK2
+// RUN: %env_asan_opts=handle_segv=2:allow_user_segv_handler=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK2
 
-// RUN: %clangxx_asan -O0 %s -o %t && %env_asan_opts=handle_segv=0:allow_user_segv_handler=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK0
-// RUN: %clangxx_asan -O0 %s -o %t && %env_asan_opts=handle_segv=1:allow_user_segv_handler=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK2
-// RUN: %clangxx_asan -O0 %s -o %t && %env_asan_opts=handle_segv=2:allow_user_segv_handler=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK2
-
-// RUN: %clangxx_asan -O0 %s -o %t && %env_asan_opts=handle_segv=0:allow_user_segv_handler=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK0
-// RUN: %clangxx_asan -O0 %s -o %t && %env_asan_opts=handle_segv=1:allow_user_segv_handler=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK1
-// RUN: %clangxx_asan -O0 %s -o %t && %env_asan_opts=handle_segv=2:allow_user_segv_handler=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK2
+// RUN: %env_asan_opts=handle_segv=0:allow_user_segv_handler=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK0
+// RUN: %env_asan_opts=handle_segv=1:allow_user_segv_handler=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK1
+// RUN: %env_asan_opts=handle_segv=2:allow_user_segv_handler=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK2
+// clang-format on
 
 #include <signal.h>
 #include <stdio.h>
diff --git a/test/profile/Linux/counter_promo_nest.c b/test/profile/Linux/counter_promo_nest.c
new file mode 100644
index 000000000000..0792f0c76abb
--- /dev/null
+++ b/test/profile/Linux/counter_promo_nest.c
@@ -0,0 +1,48 @@
+// RUN: rm -fr %t.promo.prof
+// RUN: rm -fr %t.nopromo.prof
+// RUN: %clang_pgogen=%t.promo.prof/ -o %t.promo.gen -O2 %s
+// RUN: %clang_pgogen=%t.promo.prof/ -o %t.promo.gen.ll -emit-llvm -S -O2 %s
+// RUN: cat %t.promo.gen.ll | FileCheck --check-prefix=PROMO %s
+// RUN: %run %t.promo.gen
+// RUN: llvm-profdata merge -o %t.promo.profdata %t.promo.prof/
+// RUN: llvm-profdata show --counts --all-functions %t.promo.profdata  > %t.promo.dump
+// RUN: %clang_pgogen=%t.nopromo.prof/ -mllvm -do-counter-promotion=false -o %t.nopromo.gen -O2 %s
+// RUN: %run %t.nopromo.gen
+// RUN: llvm-profdata merge -o %t.nopromo.profdata %t.nopromo.prof/
+// RUN: llvm-profdata show --counts --all-functions %t.nopromo.profdata  > %t.nopromo.dump
+// RUN: diff %t.promo.profdata %t.nopromo.profdata
+int g;
+__attribute__((noinline)) void bar() {
+ g++;
+}
+
+extern int printf(const char*,...);
+
+int c = 10;
+
+int main()
+// PROMO-LABEL: @main
+// PROMO: load{{.*}}@__profc_main{{.*}}
+// PROMO-NEXT: add
+// PROMO-NEXT: store{{.*}}@__profc_main{{.*}}
+// PROMO-NEXT: load{{.*}}@__profc_main{{.*}}
+// PROMO-NEXT: add
+// PROMO-NEXT: store{{.*}}@__profc_main{{.*}}
+{
+  int i, j, k;
+
+  g = 0;
+  for (i = 0; i < c; i++)
+    for (j = 0; j < c; j++)
+       for (k = 0; k < c; k++)
+           bar();
+
+  for (i = 0; i < c; i++)
+    for (j = 0; j < 10*c;j++)
+        bar();
+
+  for (i = 0; i < 100*c; i++)
+    bar();
+
+  return 0;
+}
diff --git a/test/sanitizer_common/ios_commands/iossim_run.py b/test/sanitizer_common/ios_commands/iossim_run.py
index 732880f35183..47b847f5329c 100755
--- a/test/sanitizer_common/ios_commands/iossim_run.py
+++ b/test/sanitizer_common/ios_commands/iossim_run.py
@@ -8,8 +8,9 @@ if not "SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER" in os.environ:
 
 device_id = os.environ["SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER"]
 
-if "ASAN_OPTIONS" in os.environ:
-    os.environ["SIMCTL_CHILD_ASAN_OPTIONS"] = os.environ["ASAN_OPTIONS"]
+for e in ["ASAN_OPTIONS", "TSAN_OPTIONS"]:
+  if e in os.environ:
+    os.environ["SIMCTL_CHILD_" + e] = os.environ[e]
 
 exitcode = subprocess.call(["xcrun", "simctl", "spawn", device_id] + sys.argv[1:])
 if exitcode > 125:
diff --git a/test/tsan/CMakeLists.txt b/test/tsan/CMakeLists.txt
index 2db6ce0a8c1a..a68908612952 100644
--- a/test/tsan/CMakeLists.txt
+++ b/test/tsan/CMakeLists.txt
@@ -1,3 +1,5 @@
+set(TSAN_LIT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+
 set(TSAN_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS})
 if(${COMPILER_RT_DEFAULT_TARGET_ARCH} MATCHES "x86_64")
   list(APPEND TSAN_TEST_DEPS GotsanRuntimeCheck)
@@ -22,6 +24,11 @@ if(APPLE)
 endif()
 
 foreach(arch ${TSAN_TEST_ARCH})
+  set(TSAN_TEST_IOS "0")
+  pythonize_bool(TSAN_TEST_IOS)
+  set(TSAN_TEST_IOSSIM "0")
+  pythonize_bool(TSAN_TEST_IOSSIM)
+
   set(TSAN_TEST_TARGET_ARCH ${arch})
   string(TOLOWER "-${arch}" TSAN_TEST_CONFIG_SUFFIX)
   get_test_cc_for_arch(${arch} TSAN_TEST_TARGET_CC TSAN_TEST_TARGET_CFLAGS)
@@ -35,6 +42,53 @@ foreach(arch ${TSAN_TEST_ARCH})
   list(APPEND TSAN_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME})
 endforeach()
 
+# iOS and iOS simulator test suites
+# These are not added into "check-all", in order to run these tests, use
+# "check-tsan-iossim-x86_64" and similar. They also require an extra environment
+# variable to select which iOS device or simulator to use, e.g.:
+# SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER="iPhone 6"
+if(APPLE)
+  set(EXCLUDE_FROM_ALL ON)
+
+  set(TSAN_TEST_TARGET_CC ${COMPILER_RT_TEST_COMPILER})
+  set(TSAN_TEST_IOS "1")
+  pythonize_bool(TSAN_TEST_IOS)
+
+  set(arch "x86_64")
+  set(TSAN_TEST_IOSSIM "1")
+  pythonize_bool(TSAN_TEST_IOSSIM)
+  set(TSAN_TEST_TARGET_ARCH ${arch})
+  set(TSAN_TEST_TARGET_CFLAGS "-arch ${arch} -isysroot ${DARWIN_iossim_SYSROOT} ${COMPILER_RT_TEST_COMPILER_CFLAGS}")
+  set(TSAN_TEST_CONFIG_SUFFIX "-${arch}-iossim")
+  string(TOUPPER ${arch} ARCH_UPPER_CASE)
+  set(CONFIG_NAME "IOSSim${ARCH_UPPER_CASE}Config")
+  configure_lit_site_cfg(
+    ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
+    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg
+    )
+  add_lit_testsuite(check-tsan-iossim-${arch} "ThreadSanitizer iOS Simulator ${arch} tests"
+    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/
+    DEPENDS ${TSAN_TEST_DEPS})
+
+  set(arch "arm64")
+  set(TSAN_TEST_IOSSIM "0")
+  pythonize_bool(TSAN_TEST_IOSSIM)
+  set(TSAN_TEST_TARGET_ARCH ${arch})
+  set(TSAN_TEST_TARGET_CFLAGS "-arch ${arch} -isysroot ${DARWIN_ios_SYSROOT} ${COMPILER_RT_TEST_COMPILER_CFLAGS}")
+  set(TSAN_TEST_CONFIG_SUFFIX "-${arch}-ios")
+  string(TOUPPER ${arch} ARCH_UPPER_CASE)
+  set(CONFIG_NAME "IOS${ARCH_UPPER_CASE}Config")
+  configure_lit_site_cfg(
+    ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.in
+    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg
+    )
+  add_lit_testsuite(check-tsan-ios-${arch} "ThreadSanitizer iOS Simulator ${arch} tests"
+    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/
+    DEPENDS ${TSAN_TEST_DEPS})
+
+  set(EXCLUDE_FROM_ALL OFF)
+endif()
+
 if(COMPILER_RT_INCLUDE_TESTS)
   configure_lit_site_cfg(
     ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.in
diff --git a/test/tsan/Darwin/dlopen.cc b/test/tsan/Darwin/dlopen.cc
index 7382a6de28c5..3d12b815f9c2 100644
--- a/test/tsan/Darwin/dlopen.cc
+++ b/test/tsan/Darwin/dlopen.cc
@@ -4,6 +4,8 @@
 
 // REQUIRES: osx-autointerception
 
+// XFAIL: ios
+
 // RUN: %clangxx_tsan %s -o %t.so -shared -DSHARED_LIB
 // RUN: %clangxx_tsan -fno-sanitize=thread %s -o %t
 
diff --git a/test/tsan/Darwin/ignore-noninstrumented.mm b/test/tsan/Darwin/ignore-noninstrumented.mm
index 528e07b9a721..668a76a462ce 100644
--- a/test/tsan/Darwin/ignore-noninstrumented.mm
+++ b/test/tsan/Darwin/ignore-noninstrumented.mm
@@ -1,4 +1,4 @@
-// Check that ignore_noninstrumented_modules=1 supresses races from system libraries on OS X.
+// Check that ignore_noninstrumented_modules=1 suppresses races from system libraries on OS X.
 
 // RUN: %clang_tsan %s -o %t -framework Foundation
 
diff --git a/test/tsan/Darwin/ignored-interceptors.mm b/test/tsan/Darwin/ignored-interceptors.mm
index 1105132a3cb6..b2e40f07d574 100644
--- a/test/tsan/Darwin/ignored-interceptors.mm
+++ b/test/tsan/Darwin/ignored-interceptors.mm
@@ -1,4 +1,4 @@
-// Check that ignore_interceptors_accesses=1 supresses reporting races from
+// Check that ignore_interceptors_accesses=1 suppresses reporting races from
 // system libraries on OS X. There are currently false positives coming from
 // libxpc, libdispatch, CoreFoundation and others, because these libraries use
 // TSan-invisible atomics as synchronization.
diff --git a/test/tsan/Darwin/osspinlock-norace.cc b/test/tsan/Darwin/osspinlock-norace.cc
index 2ac3989c223e..5de02c225f0b 100644
--- a/test/tsan/Darwin/osspinlock-norace.cc
+++ b/test/tsan/Darwin/osspinlock-norace.cc
@@ -1,8 +1,12 @@
 // RUN: %clangxx_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s
-#include <libkern/OSAtomic.h>
 #include <pthread.h>
+#include <stdint.h>
 #include <stdio.h>
 
+typedef int32_t OSSpinLock;
+extern "C" void OSSpinLockLock(OSSpinLock *);
+extern "C" void OSSpinLockUnlock(OSSpinLock *);
+
 int Global;
 OSSpinLock lock;
 
diff --git a/test/tsan/Darwin/signals-blocked.cc b/test/tsan/Darwin/signals-blocked.cc
new file mode 100644
index 000000000000..209dc2229ff6
--- /dev/null
+++ b/test/tsan/Darwin/signals-blocked.cc
@@ -0,0 +1,75 @@
+// RUN: %clangxx_tsan %s -o %t && %run %t 2>&1 | FileCheck %s
+
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <unistd.h>
+#include <errno.h>
+
+volatile bool signal_delivered;
+
+static void handler(int sig) {
+  if (sig == SIGALRM)
+    signal_delivered = true;
+}
+
+static void* thr(void *p) {
+  sigset_t sigset;
+  sigemptyset(&sigset);
+  sigaddset(&sigset, SIGALRM);
+  int ret = pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
+  if (ret) abort();
+
+  struct sigaction act = {};
+  act.sa_handler = &handler;
+  if (sigaction(SIGALRM, &act, 0)) {
+    perror("sigaction");
+    exit(1);
+  }
+
+  itimerval t;
+  t.it_value.tv_sec = 0;
+  t.it_value.tv_usec = 10000;
+  t.it_interval = t.it_value;
+  if (setitimer(ITIMER_REAL, &t, 0)) {
+    perror("setitimer");
+    exit(1);
+  }
+
+  while (!signal_delivered) {
+    usleep(1000);
+  }
+
+  t.it_value.tv_usec = 0;
+  if (setitimer(ITIMER_REAL, &t, 0)) {
+    perror("setitimer");
+    exit(1);
+  }
+
+  fprintf(stderr, "SIGNAL DELIVERED\n");
+
+  return 0;
+}
+
+int main() {
+  sigset_t sigset;
+  sigemptyset(&sigset);
+  sigaddset(&sigset, SIGALRM);
+  int ret = pthread_sigmask(SIG_BLOCK, &sigset, NULL);
+  if (ret) abort();
+
+  pthread_t th;
+  pthread_create(&th, 0, thr, 0);
+  pthread_join(th, 0);
+
+  fprintf(stderr, "DONE\n");
+  return 0;
+}
+
+// CHECK-NOT: WARNING: ThreadSanitizer:
+// CHECK: SIGNAL DELIVERED
+// CHECK: DONE
+// CHECK-NOT: WARNING: ThreadSanitizer:
diff --git a/test/tsan/Darwin/xpc-cancel.mm b/test/tsan/Darwin/xpc-cancel.mm
index 91dafc3eadda..ac7aed08c16c 100644
--- a/test/tsan/Darwin/xpc-cancel.mm
+++ b/test/tsan/Darwin/xpc-cancel.mm
@@ -1,7 +1,7 @@
 // RUN: %clang_tsan %s -o %t -framework Foundation
 // RUN: %run %t 2>&1 | FileCheck %s
 
-// XFAIL: ios
+// UNSUPPORTED: ios
 
 #import <Foundation/Foundation.h>
 #import <xpc/xpc.h>
diff --git a/test/tsan/Darwin/xpc-race.mm b/test/tsan/Darwin/xpc-race.mm
index 2e965e4a0a1c..a1e214c12b7d 100644
--- a/test/tsan/Darwin/xpc-race.mm
+++ b/test/tsan/Darwin/xpc-race.mm
@@ -1,7 +1,7 @@
 // RUN: %clang_tsan %s -o %t -framework Foundation
 // RUN: %deflake %run %t 2>&1 | FileCheck %s
 
-// XFAIL: ios
+// UNSUPPORTED: ios
 
 #import <Foundation/Foundation.h>
 #import <xpc/xpc.h>
diff --git a/test/tsan/Darwin/xpc.mm b/test/tsan/Darwin/xpc.mm
index c5e78a5779e0..036841ed7121 100644
--- a/test/tsan/Darwin/xpc.mm
+++ b/test/tsan/Darwin/xpc.mm
@@ -1,7 +1,7 @@
 // RUN: %clang_tsan %s -o %t -framework Foundation
 // RUN: %run %t 2>&1 | FileCheck %s
 
-// XFAIL: ios
+// UNSUPPORTED: ios
 
 #import <Foundation/Foundation.h>
 #import <xpc/xpc.h>
diff --git a/test/tsan/deep_stack1.cc b/test/tsan/deep_stack1.cc
index 39185efee7a9..44dd0c443920 100644
--- a/test/tsan/deep_stack1.cc
+++ b/test/tsan/deep_stack1.cc
@@ -24,6 +24,10 @@ void *Thread(void *p) {
   return 0;
 }
 
+static size_t RoundUp(size_t n, size_t to) {
+  return ((n + to - 1) / to) * to;
+}
+
 int main() {
   barrier_init(&barrier, 2);
   N = 50000;
@@ -31,7 +35,10 @@ int main() {
   pthread_t t;
   pthread_attr_t a;
   pthread_attr_init(&a);
-  pthread_attr_setstacksize(&a, N * 256 + (1 << 20));
+  size_t stack_size = N * 256 + (1 << 20);
+  stack_size = RoundUp(stack_size, 0x10000);  // round the stack size to 64k
+  int ret = pthread_attr_setstacksize(&a, stack_size);
+  if (ret) abort();
   pthread_create(&t, &a, Thread, 0);
 #ifdef ORDER2
   barrier_wait(&barrier);
diff --git a/test/tsan/ignore_lib0.cc b/test/tsan/ignore_lib0.cc
index d6ae72f31638..84632019fccb 100644
--- a/test/tsan/ignore_lib0.cc
+++ b/test/tsan/ignore_lib0.cc
@@ -11,6 +11,8 @@
 // Some aarch64 kernels do not support non executable write pages
 // REQUIRES: stable-runtime
 
+// UNSUPPORTED: ios
+
 #ifndef LIB
 
 extern "C" void libfunc();
diff --git a/test/tsan/java_find.cc b/test/tsan/java_find.cc
new file mode 100644
index 000000000000..078aac520218
--- /dev/null
+++ b/test/tsan/java_find.cc
@@ -0,0 +1,69 @@
+// RUN: %clangxx_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s
+#include "java.h"
+
+int const kHeapSize = 1024 * 1024;
+
+static void verify_find(jptr from, jptr to, jptr expected_addr,
+                        jptr expected_size) {
+  jptr addr = from;
+  jptr size = __tsan_java_find(&addr, to);
+  if (expected_size) {
+    if (!size) {
+      fprintf(stderr, "FAILED: range: [%p..%p): found nothing\n", (void *)from,
+              (void *)to);
+      return;
+    } else if (expected_size != size) {
+      fprintf(stderr, "FAILED: range: [%p..%p): wrong size, %lu instead of %lu\n",
+              (void *)from, (void *)to, size, expected_size);
+      return;
+    }
+  } else if (size) {
+    fprintf(stderr,
+            "FAILED: range [%p..%p): did not expect to find anything here\n",
+            (void *)from, (void *)to);
+    return;
+  } else {
+    return;
+  }
+  if (expected_addr != addr) {
+    fprintf(
+        stderr,
+        "FAILED: range [%p..%p): expected to find object at %p, found at %p\n",
+        (void *)from, (void *)to, (void *)expected_addr, (void *)addr);
+  }
+}
+
+int main() {
+  const jptr jheap = (jptr)malloc(kHeapSize + 8) + 8;
+  const jptr jheap_end = jheap + kHeapSize;
+  __tsan_java_init(jheap, kHeapSize);
+  const jptr addr1 = jheap;
+  const int size1 = 16;
+  __tsan_java_alloc(jheap, size1);
+
+  const jptr addr2 = addr1 + size1;
+  const int size2 = 32;
+  __tsan_java_alloc(jheap + size1, size2);
+
+  const jptr addr3 = addr2 + size2;
+  const int size3 = 1024;
+  __tsan_java_alloc(jheap + size1 + size2, size3);
+
+  const jptr addr4 = addr3 + size3;
+
+  verify_find(jheap, jheap_end, addr1, size1);
+  verify_find(jheap + 8, jheap_end, addr2, size2);
+  verify_find(addr2 + 8, jheap_end, addr3, size3);
+  verify_find(addr3 + 8, jheap_end, 0, 0);
+
+  __tsan_java_move(addr2, addr4, size2);
+  verify_find(jheap + 8, jheap_end, addr3, size3);
+  verify_find(addr3 + 8, jheap_end, addr4, size2);
+  verify_find(addr4 + 8, jheap_end, 0, 0);
+
+  fprintf(stderr, "DONE\n");
+  return 0;
+}
+
+// CHECK-NOT: FAILED
+// CHECK: DONE
diff --git a/test/tsan/lit.cfg b/test/tsan/lit.cfg
index 3c98d1fdca78..0ab62db0907f 100644
--- a/test/tsan/lit.cfg
+++ b/test/tsan/lit.cfg
@@ -66,7 +66,7 @@ if config.has_libcxx and config.host_os != 'Darwin':
                           "-Wl,-rpath=%s" % libcxx_libdir]
 
 def build_invocation(compile_flags):
-  return " " + " ".join([config.clang] + compile_flags) + " "
+  return " " + " ".join([config.compile_wrapper, config.clang] + compile_flags) + " "
 
 config.substitutions.append( ("%clang_tsan ", build_invocation(clang_tsan_cflags)) )
 config.substitutions.append( ("%clangxx_tsan ", build_invocation(clang_tsan_cxxflags)) )
diff --git a/test/tsan/lit.site.cfg.in b/test/tsan/lit.site.cfg.in
index a87e8d25d6b2..a215e664a5b4 100644
--- a/test/tsan/lit.site.cfg.in
+++ b/test/tsan/lit.site.cfg.in
@@ -1,7 +1,10 @@
 @LIT_SITE_CFG_IN_HEADER@
 
 config.name_suffix = "@TSAN_TEST_CONFIG_SUFFIX@"
+config.tsan_lit_source_dir = "@TSAN_LIT_SOURCE_DIR@"
 config.has_libcxx = @TSAN_HAS_LIBCXX@
+config.ios = @TSAN_TEST_IOS_PYBOOL@
+config.iossim = @TSAN_TEST_IOSSIM_PYBOOL@
 config.target_cflags = "@TSAN_TEST_TARGET_CFLAGS@"
 config.target_arch = "@TSAN_TEST_TARGET_ARCH@"